From: ToineSiebelink Date: Thu, 15 May 2025 12:08:15 +0000 (+0100) Subject: Make Groovy Performance Test Stable X-Git-Tag: 3.6.3~14^2 X-Git-Url: https://gerrit.onap.org/r/gitweb?a=commitdiff_plain;h=c4c2a6c00ea191cb7d90e5786ec8990ddb0f61df;p=cps.git Make Groovy Performance Test Stable - Disable memory assertion - Allow default 50% margin on expected time - Allow 300% margin on very fast test < 1ms - Extend resource meter method with optional exceptional margin for an unstable test - Update all time limits on plotted test to be average (rather then limit) Issue-ID: CPS-2743 Change-Id: Icdcbe01b048c2d04166684cda4bf341a204c9a1d Signed-off-by: ToineSiebelink --- diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/NcmpPerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/NcmpPerfTestBase.groovy index 11f77a5ba0..857488cfa7 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/NcmpPerfTestBase.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/NcmpPerfTestBase.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023-2024 Nordix Foundation + * Copyright (C) 2023-2025 OpenInfra Foundation Europe. All rights reserved. * Modifications Copyright (C) 2024 TechMahindra Ltd. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy index 66beb603da..f98f1c6f91 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023-2024 Nordix Foundation + * Copyright (C) 2023-2025 OpenInfra Foundation Europe. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -26,6 +26,9 @@ abstract class PerfTestBase extends CpsIntegrationSpecBase { static def LARGE_SCHEMA_SET = 'largeSchemaSet' static def PERFORMANCE_RECORD = [] + static def DEFAULT_TIME_MARGIN = 1.5 // Allow 50% margin + static def VERY_FAST_TEST_THRESHOLD = 0.01 // Defintion of a very vast test (hard to measure) + static def DEFAULT_TIME_MARGIN_FOR_VERY_FAST_TEST = 3 // Allow 200% margin on very fast test (accuracy is an issue) def cleanupSpec() { println('##################################################################################################') @@ -53,16 +56,22 @@ abstract class PerfTestBase extends CpsIntegrationSpecBase { abstract def createInitialData() - def recordAndAssertResourceUsage(String shortTitle, double thresholdInSec, double recordedTimeInSec, memoryLimit, memoryUsageInMB) { - def pass = recordedTimeInSec <= thresholdInSec && memoryUsageInMB <= memoryLimit + def recordAndAssertResourceUsage(String shortTitle, double expectedTimeInSec, double recordedTimeInSec, memoryLimit, memoryUsageInMB, double timeMargin) { + if (expectedTimeInSec <= VERY_FAST_TEST_THRESHOLD) { + timeMargin = DEFAULT_TIME_MARGIN_FOR_VERY_FAST_TEST + } + def testPassed = recordedTimeInSec <= timeMargin * expectedTimeInSec if (shortTitle.length() > 40) { shortTitle = shortTitle.substring(0, 40) } - def record = String.format('%2d.%-40s limit %8.3f took %8.3f sec %,8.2f MB used ', PERFORMANCE_RECORD.size() + 1, shortTitle, thresholdInSec, recordedTimeInSec, memoryUsageInMB) - record += pass ? 'PASS' : 'FAIL' + def record = String.format('%2d.%-40s limit %8.3f took %8.3f sec %,8.2f MB used ', PERFORMANCE_RECORD.size() + 1, shortTitle, expectedTimeInSec, recordedTimeInSec, memoryUsageInMB) + record += testPassed ? 'PASS' : 'FAIL' PERFORMANCE_RECORD.add(record) - assert recordedTimeInSec <= thresholdInSec - assert memoryUsageInMB <= memoryLimit + assert recordedTimeInSec <= timeMargin * expectedTimeInSec return true } + + def recordAndAssertResourceUsage(String shortTitle, double thresholdInSec, double recordedTimeInSec, memoryLimit, memoryUsageInMB) { + recordAndAssertResourceUsage(shortTitle, thresholdInSec, recordedTimeInSec, memoryLimit, memoryUsageInMB, DEFAULT_TIME_MARGIN) + } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy index b04dfa01b4..da34fe5a1c 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023-2024 Nordix Foundation + * Copyright (C) 2023-2025 OpenInfra Foundation Europe. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -47,7 +47,7 @@ class CpsDataServiceLimitsPerfTest extends CpsPerfTestBase { resourceMeter.stop() def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'the operation completes within 12 seconds' - recordAndAssertResourceUsage("Creating 33,000 books", 18.891, durationInSeconds, 150, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Creating 33,000 books", 12, durationInSeconds, 150, resourceMeter.getTotalMemoryUsageInMB()) } def 'Get data nodes from multiple xpaths 32K (2^15) limit exceeded.'() { @@ -81,14 +81,14 @@ class CpsDataServiceLimitsPerfTest extends CpsPerfTestBase { } def 'Clean up test data.'() { - when: + when: 'cleaning up all performance test data nodes and anchors.' resourceMeter.start() cpsDataService.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'limitsAnchor', OffsetDateTime.now()) cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, 'limitsAnchor') resourceMeter.stop() def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'test data is deleted in 1 second' - recordAndAssertResourceUsage("Deleting test data", 0.141, durationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Deleting test data", 0.056, durationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB()) } def countDataNodes() { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy index c68806bdff..5d7b58cf54 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023-2024 Nordix Foundation + * Copyright (C) 2023-2025 OpenInfra Foundation Europe. All rights reserved. * Modifications Copyright (C) 2025 TechMahindra Ltd. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); @@ -57,7 +57,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete 100 containers', 3.9, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete 100 containers', 1.5, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch delete 100 container nodes'() { @@ -71,7 +71,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Batch delete 100 containers', 0.87, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Batch delete 100 containers', 0.7, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete 100 list elements'() { @@ -87,7 +87,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete 100 lists elements', 4.22, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete 100 lists elements', 1.5, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch delete 100 list elements'() { @@ -101,7 +101,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Batch delete 100 lists elements', 0.87, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Batch delete 100 lists elements', 0.7, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete 100 whole lists'() { @@ -117,7 +117,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete 100 whole lists', 5.4, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete 100 whole lists', 3.2, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB(), 3) } def 'Batch delete 100 whole lists'() { @@ -131,7 +131,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Batch delete 100 whole lists', 1.98, deleteDurationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Batch delete 100 whole lists', 3.4, deleteDurationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB(), 3) } def 'Delete 1 large data node'() { @@ -141,7 +141,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete one large node', 2.35, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete one large node', 2, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete root node with many descendants'() { @@ -151,7 +151,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete root node', 2.23, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete root node', 2, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete data nodes for an anchor'() { @@ -161,7 +161,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete data nodes for anchor', 2.25, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete data nodes for anchor', 2, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch delete 100 non-existing nodes'() { @@ -175,7 +175,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Batch delete 100 non-existing', 0.74, deleteDurationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Batch delete 100 non-existing', 2.6, deleteDurationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB()) } def 'Clean up test data'() { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy index 5da2dbf8d8..3d6437ffdb 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023 Nordix Foundation + * Copyright (C) 2023-2025 OpenInfra Foundation Europe. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -45,8 +45,8 @@ class GetPerfTest extends CpsPerfTestBase { where: 'the following parameters are used' scenario | fetchDescendantsOption || durationLimit | memoryLimit | expectedNumberOfDataNodes 'no descendants' | OMIT_DESCENDANTS || 0.01 | 1 | 1 - 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.06 | 5 | 1 + OPENROADM_DEVICES_PER_ANCHOR - 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.47 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.03 | 5 | 1 + OPENROADM_DEVICES_PER_ANCHOR + 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.1 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } def 'Read data trees for multiple xpaths'() { @@ -60,7 +60,7 @@ class GetPerfTest extends CpsPerfTestBase { then: 'requested nodes and their descendants are returned' assert countDataNodesInTree(result) == OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE and: 'all data is read within expected time and memory used is within limit' - recordAndAssertResourceUsage("Read datatrees for multiple xpaths", 2.2, durationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Read datatrees for multiple xpaths", 0.55, durationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB(),3) } def 'Read for multiple xpaths to non-existing datanodes'() { @@ -88,9 +88,9 @@ class GetPerfTest extends CpsPerfTestBase { recordAndAssertResourceUsage("Read datatrees using ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following xpaths are used' scenario | xpath || durationLimit | memoryLimit | expectedNumberOfDataNodes - 'openroadm root' | '/' || 1.28 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE - 'openroadm top element' | '/openroadm-devices' || 1.3 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE - 'openroadm whole list' | '/openroadm-devices/openroadm-device' || 1.51 | 250 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'openroadm root' | '/' || 1 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'openroadm top element' | '/openroadm-devices' || 0.9 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'openroadm whole list' | '/openroadm-devices/openroadm-device' || 1.06 | 250 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy index e52d3f819c..8586926403 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/ModuleQueryPerfTest.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2024-2025 Nordix Foundation + * Copyright (C) 2024-2025 OpenInfra Foundation Europe. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -70,9 +70,7 @@ class ModuleQueryPerfTest extends CpsPerfTestBase { } resourceMeter.stop() then: 'operation takes less than expected duration' - recordAndAssertResourceUsage('Module query test setup', - 45, resourceMeter.totalTimeInSeconds, - 500, resourceMeter.totalMemoryUsageInMB + recordAndAssertResourceUsage('Module query test setup', 90, resourceMeter.totalTimeInSeconds, 500, resourceMeter.totalMemoryUsageInMB ) } @@ -84,13 +82,11 @@ class ModuleQueryPerfTest extends CpsPerfTestBase { then: 'expected number of anchors is returned' assert result.size() == TOTAL_TEST_ANCHORS and: 'operation completes with expected resource usage' - recordAndAssertResourceUsage("Query for anchors with ${scenario}", - expectedTimeInSeconds, resourceMeter.totalTimeInSeconds, - 5, resourceMeter.totalMemoryUsageInMB) + recordAndAssertResourceUsage("Query for anchors with ${scenario}", 0.1, resourceMeter.totalTimeInSeconds, 5, resourceMeter.totalMemoryUsageInMB) where: 'the following parameters are used' - scenario | yangModuleName || expectedTimeInSeconds - '1 KB module' | 'module0' || 0.05 - '1000 KB module' | 'module1' || 0.05 + scenario | yangModuleName + '1 KB module' | 'module0' + '1000 KB module' | 'module1' } def 'Module query - Clean up test data.'() { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy index 8c429b3a30..bb8143c8c1 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy @@ -45,10 +45,10 @@ class QueryPerfTest extends CpsPerfTestBase { recordAndAssertResourceUsage("Query 1 anchor ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' scenario | cpsPath || durationLimit | memoryLimit | expectedNumberOfDataNodes - 'top element' | '/openroadm-devices' || 1.27 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 + 'top element' | '/openroadm-devices' || 0.89 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || 1.3 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE - 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 1.46 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 - 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 1.32 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 + 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 1.1 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 + 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 0.93 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 'non-existing data' | '/path/to/non-existing/node[@id="1"]' || 0.01 | 1 | 0 } @@ -64,10 +64,10 @@ class QueryPerfTest extends CpsPerfTestBase { recordAndAssertResourceUsage("Query across anchors ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' scenario | cpspath || durationLimit | memoryLimit | expectedNumberOfDataNodes - 'top element' | '/openroadm-devices' || 3.76 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) + 'top element' | '/openroadm-devices' || 3.0 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || 3.3 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE) - 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 3.96 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) - 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 3.76 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) + 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 3.2 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) + 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 3.0 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) } def 'Query with leaf condition and #scenario.'() { @@ -82,9 +82,9 @@ class QueryPerfTest extends CpsPerfTestBase { recordAndAssertResourceUsage("Query with ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' scenario | fetchDescendantsOption || durationLimit | memoryLimit | expectedNumberOfDataNodes - 'no descendants' | OMIT_DESCENDANTS || 0.1 | 6 | OPENROADM_DEVICES_PER_ANCHOR - 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.16 | 12 | OPENROADM_DEVICES_PER_ANCHOR * 2 - 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.4 | 200 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'no descendants' | OMIT_DESCENDANTS || 0.08 | 6 | OPENROADM_DEVICES_PER_ANCHOR + 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.09 | 12 | OPENROADM_DEVICES_PER_ANCHOR * 2 + 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.05 | 200 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } def 'Query ancestors with #scenario.'() { @@ -99,9 +99,9 @@ class QueryPerfTest extends CpsPerfTestBase { recordAndAssertResourceUsage("Query ancestors with ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' scenario | fetchDescendantsOption || durationLimit | memoryLimit | expectedNumberOfDataNodes - 'no descendants' | OMIT_DESCENDANTS || 0.09 | 3 | 1 - 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.11 | 8 | 1 + OPENROADM_DEVICES_PER_ANCHOR - 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.34 | 400 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'no descendants' | OMIT_DESCENDANTS || 0.06 | 3 | 1 + 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.07 | 8 | 1 + OPENROADM_DEVICES_PER_ANCHOR + 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.0 | 400 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } def 'Query data leaf with #scenario.'() { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy index 69edb2034a..9d09439b04 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023-2024 Nordix Foundation + * Copyright (C) 2023-2025 OpenInfra Foundation Europe. All rights reserved. * Modifications Copyright (C) 2024 TechMahindra Ltd. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); @@ -76,17 +76,15 @@ class UpdatePerfTest extends CpsPerfTestBase { assert totalNodes == countDataNodes(changeLeaves? '/openroadm-devices/openroadm-device[@status="fail"]' : '/openroadm-devices/openroadm-device[@status="success"]') and: 'update completes within expected time and memory used is within limit' - recordAndAssertResourceUsage(scenario, - timeLimit, resourceMeter.getTotalTimeInSeconds(), - memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage(scenario, expectedTime, resourceMeter.getTotalTimeInSeconds(), memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: - scenario | totalNodes | startId | changeLeaves || timeLimit | memoryLimit - 'Replace 0 nodes with 100' | 100 | 1 | false || 3.99 | 200 - 'Replace 100 using same data' | 100 | 1 | false || 7.46 | 200 - 'Replace 100 with new leaf values' | 100 | 1 | true || 7.87 | 200 - 'Replace 100 with 100 new nodes' | 100 | 101 | false || 13.85 | 200 - 'Replace 50 existing and 50 new' | 100 | 151 | true || 10.82 | 200 - 'Replace 100 nodes with 0' | 0 | 1 | false || 8.91 | 200 + scenario | totalNodes | startId | changeLeaves || expectedTime | memoryLimit + 'Replace 0 nodes with 100' | 100 | 1 | false || 3.0 | 200 + 'Replace 100 using same data' | 100 | 1 | false || 2.3 | 200 + 'Replace 100 with new leaf values' | 100 | 1 | true || 2.3 | 200 + 'Replace 100 with 100 new nodes' | 100 | 101 | false || 13.85 | 200 + 'Replace 50 existing and 50 new' | 100 | 151 | true || 4.2 | 200 + 'Replace 100 nodes with 0' | 0 | 1 | false || 3.5 | 200 } def 'Replace list content: #scenario.'() { @@ -103,16 +101,16 @@ class UpdatePerfTest extends CpsPerfTestBase { : '/openroadm-devices/openroadm-device[@status="success"]') and: 'update completes within expected time and memory used is within limit' recordAndAssertResourceUsage(scenario, - timeLimit, resourceMeter.getTotalTimeInSeconds(), + expectedTime, resourceMeter.getTotalTimeInSeconds(), memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: - scenario | totalNodes | startId | changeLeaves || timeLimit | memoryLimit - 'Replace list of 0 with 100' | 100 | 1 | false || 4.01 | 200 - 'Replace list of 100 using same data' | 100 | 1 | false || 5.53 | 200 - 'Replace list of 100 with new leaf values' | 100 | 1 | true || 6.96 | 200 - 'Replace list with 100 new nodes' | 100 | 101 | false || 12.82 | 200 - 'Replace list with 50 existing and 50 new' | 100 | 151 | true || 10.42 | 200 - 'Replace list of 100 nodes with 1' | 1 | 1 | false || 9.26 | 200 + scenario | totalNodes | startId | changeLeaves || expectedTime | memoryLimit + 'Replace list of 0 with 100' | 100 | 1 | false || 3.0 | 200 + 'Replace list of 100 using same data' | 100 | 1 | false || 3.32 | 200 + 'Replace list of 100 with new leaf values' | 100 | 1 | true || 4.2 | 200 + 'Replace list with 100 new nodes' | 100 | 101 | false || 6.1 | 200 + 'Replace list with 50 existing and 50 new' | 100 | 151 | true || 4.0 | 200 + 'Replace list of 100 nodes with 1' | 1 | 1 | false || 3.34 | 200 } def 'Update leaves for 100 data nodes.'() { @@ -128,9 +126,7 @@ class UpdatePerfTest extends CpsPerfTestBase { then: 'data leaves have expected values' assert 100 == countDataNodes('/openroadm-devices/openroadm-device[@status="fail"]') and: 'update completes within expected time and memory used is within limit' - recordAndAssertResourceUsage('Update leaves for 100 data nodes', - 0.35, resourceMeter.getTotalTimeInSeconds(), - 120, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Update leaves for 100 data nodes', 0.14, resourceMeter.getTotalTimeInSeconds(), 120, resourceMeter.getTotalMemoryUsageInMB()) } def 'Clean up for CPS Update API.'() { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy index 9f6c78d5f5..7c7a961f7c 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023-2024 Nordix Foundation + * Copyright (C) 2023-2025 OpenInfra Foundation Europe. All rights reserved. * Modifications Copyright (C) 2024 TechMahindra Ltd. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); @@ -47,10 +47,10 @@ class WritePerfTest extends CpsPerfTestBase { cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR) where: totalNodes || expectedDuration | memoryLimit - 50 || 1.98 | 100 - 100 || 3.84 | 200 - 200 || 8.6 | 400 - 400 || 16.37 | 500 + 50 || 1.45 | 100 + 100 || 2.9 | 200 + 200 || 5.6 | 400 + 400 || 11.4 | 500 } def 'Writing bookstore data has exponential time.'() { @@ -72,10 +72,10 @@ class WritePerfTest extends CpsPerfTestBase { cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR) where: totalBooks || expectedDuration | memoryLimit - 800 || 0.38 | 50 - 1600 || 0.95 | 100 - 3200 || 2.71 | 150 - 6400 || 8.08 | 200 + 800 || 0.26 | 50 + 1600 || 0.75 | 100 + 3200 || 2.2 | 150 + 6400 || 6.9 | 200 } def 'Writing openroadm list data using saveListElements.'() { @@ -100,10 +100,10 @@ class WritePerfTest extends CpsPerfTestBase { cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR) where: totalNodes || expectedDuration | memoryLimit - 50 || 1.8 | 100 - 100 || 3.93 | 200 - 200 || 7.77 | 400 - 400 || 16.59 | 500 + 50 || 1.35 | 100 + 100 || 2.7 | 200 + 200 || 5.4 | 400 + 400 || 10.8 | 500 } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy index 08b186ab3c..b4456e3b26 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023 Nordix Foundation + * Copyright (C) 2023-2025 OpenInfra Foundation Europe. All rights reserved. * Modifications Copyright (C) 2024 TechMahindra Ltd. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); @@ -97,7 +97,7 @@ class CmDataSubscriptionsPerfTest extends NcmpPerfTestBase { def resultAfter = objectUnderTest.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_ANCHOR, cpsPath, INCLUDE_ALL_DESCENDANTS) assert resultAfter.collect {it.leaves.subscribers.size()}.sum() == totalNumberOfEntries * (1 + numberOfCmDataSubscribers) and: 'update matching subscription within 15 seconds' - recordAndAssertResourceUsage("Update matching subscription", 13.86, durationInSeconds, 1000, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Update matching subscription", 6.2, durationInSeconds, 1000, resourceMeter.getTotalMemoryUsageInMB()) } def 'Worst case new subscription (200x10 new entries).'() { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy index dbf7e71710..27ec66f30d 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023-2025 Nordix Foundation + * Copyright (C) 2023-2025 OpenInfra Foundation Europe. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -43,16 +43,12 @@ class CmHandleQueryPerfTest extends NcmpPerfTestBase { def iterations = 2500 // set this to 15000 for very accurate results (but test takes much longer) resourceMeter.start() (1..iterations).forEach { - cpsDataService.getDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR, - '/dmi-registry/cm-handles[@id="cm-' + it + '"]', OMIT_DESCENDANTS) - objectUnderTest.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR, - '/dmi-registry/cm-handles[@alternate-id="alt=' + it + '"]', OMIT_DESCENDANTS) + cpsDataService.getDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR, '/dmi-registry/cm-handles[@id="cm-' + it + '"]', OMIT_DESCENDANTS) + objectUnderTest.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR, '/dmi-registry/cm-handles[@alternate-id="alt=' + it + '"]', OMIT_DESCENDANTS) } resourceMeter.stop() then: 'resource usage is as expected' - recordAndAssertResourceUsage('JVM warmup for CmHandleQueryPerfTest', - 30, resourceMeter.totalTimeInSeconds, - 300, resourceMeter.totalMemoryUsageInMB) + recordAndAssertResourceUsage('JVM warmup for CmHandleQueryPerfTest', 60, resourceMeter.totalTimeInSeconds, 300, resourceMeter.totalMemoryUsageInMB) } def 'Query CM Handle IDs by a property name and value.'() { @@ -67,7 +63,7 @@ class CmHandleQueryPerfTest extends NcmpPerfTestBase { resourceMeter.stop() def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'the required operations are performed within required time' - recordAndAssertResourceUsage("CpsPath Registry attributes Query", 3.96, durationInSeconds, 400, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("CpsPath Registry attributes Query", 2.4, durationInSeconds, 400, resourceMeter.getTotalMemoryUsageInMB()) and: 'all nodes are returned' result.size() == TOTAL_CM_HANDLES and: 'the tree contains all the expected descendants too' @@ -87,31 +83,23 @@ class CmHandleQueryPerfTest extends NcmpPerfTestBase { assert count == 100 and: 'average performance is as expected' def averageResponseTime = resourceMeter.totalTimeInSeconds / 100 - recordAndAssertResourceUsage('Look up CM-handle by id', - expectedAverageResponseTime, averageResponseTime, - 15, resourceMeter.totalMemoryUsageInMB) - where: - expectedAverageResponseTime = 8 * MILLISECONDS + recordAndAssertResourceUsage('Look up CM-handle by id', 0.003, averageResponseTime, 15, resourceMeter.totalMemoryUsageInMB) } def 'CM-handle is looked up by alternate-id.'() { - when: 'CM-handles are looked up by alternate-id 100 times' + when: 'CM-handles are looked up by alternate-id 1000 times' int count = 0 resourceMeter.start() - (1..100).each { + (1..1000).each { count += cpsQueryService.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR, - '/dmi-registry/cm-handles[@alternate-id="alt=' + it + '"]', OMIT_DESCENDANTS).size() + '/dmi-registry/cm-handles[@alternate-id="alt=/a/b/c/d-' + it + '"]', OMIT_DESCENDANTS).size() } resourceMeter.stop() then: - assert count == 100 + assert count == 1000 and: 'average performance is as expected' - def averageResponseTime = resourceMeter.totalTimeInSeconds / 100 - recordAndAssertResourceUsage('Look up CM-handle by alternate-id', - expectedAverageResponseTime, averageResponseTime, - 15, resourceMeter.totalMemoryUsageInMB) - where: - expectedAverageResponseTime = 20 * MILLISECONDS + def averageResponseTime = resourceMeter.totalTimeInSeconds / 1000 + recordAndAssertResourceUsage('Look up CM-handle by alternate-id', 0.001, averageResponseTime, 15, resourceMeter.totalMemoryUsageInMB) } def 'A batch of CM-handles is looked up by alternate-id.'() { @@ -146,11 +134,7 @@ class CmHandleQueryPerfTest extends NcmpPerfTestBase { assert count == TOTAL_CM_HANDLES * 100 then: 'average performance is as expected' def averageResponseTime = resourceMeter.totalTimeInSeconds / 100 - recordAndAssertResourceUsage('Look up CM-handles by module-set-tag', - expectedAverageResponseTime, averageResponseTime, - 500, resourceMeter.totalMemoryUsageInMB) - where: - expectedAverageResponseTime = 438 * MILLISECONDS + recordAndAssertResourceUsage('Look up CM-handles by module-set-tag', 0.26, averageResponseTime, 500, resourceMeter.totalMemoryUsageInMB) } }