From: ToineSiebelink Date: Tue, 27 May 2025 15:38:39 +0000 (+0100) Subject: Make Groovy Performance Test Stable #6 X-Git-Tag: 3.6.3~3 X-Git-Url: https://gerrit.onap.org/r/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F65%2F140965%2F1;p=cps.git Make Groovy Performance Test Stable #6 - Fixed many expectations again now Groovy Perf does not run at same time as KPI (strangely enough most create use cases got worse) - Default margin now 100% instead of 50% - Increased volume of 1 very fast test to get more stable measurements Issue-ID:CPS-2743 Change-Id: I081d48926a9e283a429603bd78dcd79c5d8a2394 Signed-off-by: ToineSiebelink --- diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy index a617748dee..cc83bf6c9d 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy @@ -26,7 +26,7 @@ abstract class PerfTestBase extends CpsIntegrationSpecBase { static def LARGE_SCHEMA_SET = 'largeSchemaSet' static def PERFORMANCE_RECORD = [] - static def DEFAULT_TIME_LIMIT_FACTOR = 1.5 // Allow 50% margin on top of expected (average) value + static def DEFAULT_TIME_LIMIT_FACTOR = 2 // Allow 100% margin on top of expected (average) value static def VERY_FAST_TEST_THRESHOLD = 0.01 // Definition of a very vast test (hard to measure) static def DEFAULT_TIME_LIMIT_FACTOR_FOR_VERY_FAST_TEST = 3 // Allow 200% margin on very fast test (accuracy is an issue) diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy index da34fe5a1c..e242a4cc89 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy @@ -47,7 +47,7 @@ class CpsDataServiceLimitsPerfTest extends CpsPerfTestBase { resourceMeter.stop() def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'the operation completes within 12 seconds' - recordAndAssertResourceUsage("Creating 33,000 books", 12, durationInSeconds, 150, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Creating 33,000 books", 16, durationInSeconds, 150, resourceMeter.getTotalMemoryUsageInMB()) } def 'Get data nodes from multiple xpaths 32K (2^15) limit exceeded.'() { @@ -88,7 +88,7 @@ class CpsDataServiceLimitsPerfTest extends CpsPerfTestBase { resourceMeter.stop() def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'test data is deleted in 1 second' - recordAndAssertResourceUsage("Deleting test data", 0.056, durationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Deleting test data", 0.068, durationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB()) } def countDataNodes() { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy index be9252ad3e..ad98d309f4 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy @@ -57,7 +57,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is below accepted margin of the expected average' - recordAndAssertResourceUsage('Delete 100 containers', 1.4, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete 100 containers', 2.1, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch delete 100 container nodes'() { @@ -87,7 +87,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is below accepted margin of the expected average' - recordAndAssertResourceUsage('Delete 100 lists elements', 1.5, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete 100 lists elements', 2.3, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch delete 100 list elements'() { @@ -129,7 +129,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is below accepted margin (3x) of the expected average' - recordAndAssertResourceUsage('Batch delete 100 whole lists', 3.8, deleteDurationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB(), 4) + recordAndAssertResourceUsage('Batch delete 100 whole lists', 3.0, deleteDurationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB(), 4) } def 'Delete 1 large data node'() { @@ -139,7 +139,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is below accepted margin of the expected average' - recordAndAssertResourceUsage('Delete one large node', 2, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete one large node', 1.9, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete root node with many descendants'() { @@ -149,7 +149,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is below accepted margin of the expected average' - recordAndAssertResourceUsage('Delete root node', 2, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete root node', 1.9, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete data nodes for an anchor'() { @@ -159,7 +159,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is below accepted margin of the expected average' - recordAndAssertResourceUsage('Delete data nodes for anchor', 2, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete data nodes for anchor', 1.8, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch delete 100 non-existing nodes'() { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy index aa92b3940e..68c29b2203 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy @@ -40,12 +40,12 @@ class GetPerfTest extends CpsPerfTestBase { resourceMeter.stop() assert countDataNodesInTree(result) == expectedNumberOfDataNodes def durationInSeconds = resourceMeter.getTotalTimeInSeconds() - then: 'all data is read within #expectedDuration seconds and a margin of 50%' + then: 'all data is read within #expectedDuration seconds and a margin of 100%' recordAndAssertResourceUsage("Read datatrees with ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' scenario | fetchDescendantsOption || expectedDuration | memoryLimit | expectedNumberOfDataNodes 'no descendants' | OMIT_DESCENDANTS || 0.01 | 1 | 1 - 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.03 | 5 | 1 + OPENROADM_DEVICES_PER_ANCHOR + 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.05 | 5 | 1 + OPENROADM_DEVICES_PER_ANCHOR 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.1 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } @@ -60,12 +60,12 @@ class GetPerfTest extends CpsPerfTestBase { then: 'requested nodes and their descendants are returned' assert countDataNodesInTree(result) == OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE and: 'all data is read within expected time and a margin of 200%' - recordAndAssertResourceUsage("Read datatrees for multiple xpaths", 1.3, durationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB(),3) + recordAndAssertResourceUsage("Read datatrees for multiple xpaths", 2.0, durationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB(),3) } def 'Read for multiple xpaths to non-existing datanodes'() { given: 'a collection of xpaths to get' - def xpaths = (1..50).collect { "/path/to/non-existing/node[@id='" + it + "']" } + def xpaths = (1..500).collect { "/path/to/non-existing/node[@id='" + it + "']" } when: 'get data nodes from 1 anchor' resourceMeter.start() def result = objectUnderTest.getDataNodesForMultipleXpaths(CPS_PERFORMANCE_TEST_DATASPACE, 'openroadm2', xpaths, INCLUDE_ALL_DESCENDANTS) @@ -74,7 +74,7 @@ class GetPerfTest extends CpsPerfTestBase { then: 'no data is returned' assert result.isEmpty() and: 'the operation completes within within expected time' - recordAndAssertResourceUsage("Read non-existing xpaths", 0.01, durationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Read non-existing xpaths", 0.15, durationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) } def 'Read complete data trees using #scenario.'() { @@ -84,12 +84,12 @@ class GetPerfTest extends CpsPerfTestBase { assert countDataNodesInTree(result) == expectedNumberOfDataNodes resourceMeter.stop() def durationInSeconds = resourceMeter.getTotalTimeInSeconds() - then: 'all data is read within expected time and a margin of 50%' - recordAndAssertResourceUsage("Read datatrees using ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) + then: 'all data is read within expected time and a margin of 100%' + recordAndAssertResourceUsage("Read datatrees using ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB(),3) where: 'the following xpaths are used' scenario | xpath || expectedDuration | memoryLimit | expectedNumberOfDataNodes 'openroadm root' | '/' || 1 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE - 'openroadm top element' | '/openroadm-devices' || 0.9 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'openroadm top element' | '/openroadm-devices' || 1.0 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE 'openroadm whole list' | '/openroadm-devices/openroadm-device' || 1.06 | 250 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy index 8c378356f6..625cc005bd 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy @@ -41,13 +41,13 @@ class QueryPerfTest extends CpsPerfTestBase { def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'the expected number of nodes is returned' assert countDataNodesInTree(result) == expectedNumberOfDataNodes - and: 'all data is read #expectedDuration seconds with a margin of 50%' + and: 'all data is read #expectedDuration seconds with a margin of 100%' recordAndAssertResourceUsage("Query 1 anchor ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' scenario | cpsPath || expectedDuration | memoryLimit | expectedNumberOfDataNodes 'top element' | '/openroadm-devices' || 0.89 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || 1.3 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE - 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 1.1 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 + 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 1.3 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 0.93 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 'non-existing data' | '/path/to/non-existing/node[@id="1"]' || 0.01 | 1 | 0 } @@ -60,7 +60,7 @@ class QueryPerfTest extends CpsPerfTestBase { def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'the expected number of nodes is returned' assert countDataNodesInTree(result) == expectedNumberOfDataNodes - and: 'all data is read #expectedDuration seconds with a margin of 50%' + and: 'all data is read #expectedDuration seconds with a margin of 100%' recordAndAssertResourceUsage("Query across anchors ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' scenario | cpspath || expectedDuration | memoryLimit | expectedNumberOfDataNodes @@ -78,12 +78,12 @@ class QueryPerfTest extends CpsPerfTestBase { def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'the expected number of nodes is returned' assert countDataNodesInTree(result) == expectedNumberOfDataNodes - and: 'all data is read #expectedDuration seconds with a margin of 50%' + and: 'all data is read #expectedDuration seconds with a margin of 100%' recordAndAssertResourceUsage("Query with ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB(),2) where: 'the following parameters are used' scenario | fetchDescendantsOption || expectedDuration | memoryLimit | expectedNumberOfDataNodes 'no descendants' | OMIT_DESCENDANTS || 0.08 | 6 | OPENROADM_DEVICES_PER_ANCHOR - 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.09 | 12 | OPENROADM_DEVICES_PER_ANCHOR * 2 + 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.12 | 12 | OPENROADM_DEVICES_PER_ANCHOR * 2 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.05 | 200 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } @@ -95,12 +95,12 @@ class QueryPerfTest extends CpsPerfTestBase { def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'the expected number of nodes is returned' assert countDataNodesInTree(result) == expectedNumberOfDataNodes - and: 'all data is read #expectedDuration seconds with a margin of 50%' + and: 'all data is read #expectedDuration seconds with a margin of 100%' recordAndAssertResourceUsage("Query ancestors with ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' scenario | fetchDescendantsOption || expectedDuration | memoryLimit | expectedNumberOfDataNodes - 'no descendants' | OMIT_DESCENDANTS || 0.06 | 3 | 1 - 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.07 | 8 | 1 + OPENROADM_DEVICES_PER_ANCHOR + 'no descendants' | OMIT_DESCENDANTS || 0.08 | 3 | 1 + 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.09 | 8 | 1 + OPENROADM_DEVICES_PER_ANCHOR 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.0 | 400 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } @@ -112,7 +112,7 @@ class QueryPerfTest extends CpsPerfTestBase { def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'the expected number of results is returned' assert result.size() == expectedNumberOfValues - and: 'all data is read #expectedDuration seconds with a margin of 50%' + and: 'all data is read #expectedDuration seconds with a margin of 100%' recordAndAssertResourceUsage("Query data leaf ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' scenario | cpsPath || expectedDuration | memoryLimit | expectedNumberOfValues diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy index 0c66871790..c2b027d56d 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy @@ -75,16 +75,16 @@ class UpdatePerfTest extends CpsPerfTestBase { and: 'data leaves have expected values' assert totalNodes == countDataNodes(changeLeaves? '/openroadm-devices/openroadm-device[@status="fail"]' : '/openroadm-devices/openroadm-device[@status="success"]') - and: 'update completes within expected time and a margin of 50%' + and: 'update completes within expected time and a margin of 100%' recordAndAssertResourceUsage(scenario, expectedTime, resourceMeter.getTotalTimeInSeconds(), memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: scenario | totalNodes | startId | changeLeaves || expectedTime | memoryLimit - 'Replace 0 nodes with 100' | 100 | 1 | false || 2.85 | 200 - 'Replace 100 using same data' | 100 | 1 | false || 2.0 | 200 - 'Replace 100 with new leaf values' | 100 | 1 | true || 2.0 | 200 - 'Replace 100 with 100 new nodes' | 100 | 101 | false || 6.2 | 200 - 'Replace 50 existing and 50 new' | 100 | 151 | true || 4.2 | 200 - 'Replace 100 nodes with 0' | 0 | 1 | false || 3.5 | 200 + 'Replace 0 nodes with 100' | 100 | 1 | false || 3.0 | 200 + 'Replace 100 using same data' | 100 | 1 | false || 5.0 | 200 + 'Replace 100 with new leaf values' | 100 | 1 | true || 5.4 | 200 + 'Replace 100 with 100 new nodes' | 100 | 101 | false || 9.3 | 200 + 'Replace 50 existing and 50 new' | 100 | 151 | true || 7.3 | 200 + 'Replace 100 nodes with 0' | 0 | 1 | false || 6.0 | 200 } def 'Replace list content: #scenario.'() { @@ -99,18 +99,18 @@ class UpdatePerfTest extends CpsPerfTestBase { and: 'data leaves have expected values' assert totalNodes == countDataNodes(changeLeaves? '/openroadm-devices/openroadm-device[@status="fail"]' : '/openroadm-devices/openroadm-device[@status="success"]') - and: 'update completes within expected time and a margin of 50%' + and: 'update completes within expected time and a margin of 100%' recordAndAssertResourceUsage(scenario, expectedTime, resourceMeter.getTotalTimeInSeconds(), memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: scenario | totalNodes | startId | changeLeaves || expectedTime | memoryLimit 'Replace list of 0 with 100' | 100 | 1 | false || 3.0 | 200 - 'Replace list of 100 using same data' | 100 | 1 | false || 2.0 | 200 - 'Replace list of 100 with new leaf values' | 100 | 1 | true || 2.0 | 200 - 'Replace list with 100 new nodes' | 100 | 101 | false || 6.1 | 200 - 'Replace list with 50 existing and 50 new' | 100 | 151 | true || 4.0 | 200 - 'Replace list of 100 nodes with 1' | 1 | 1 | false || 3.34 | 200 + 'Replace list of 100 using same data' | 100 | 1 | false || 5.5 | 200 + 'Replace list of 100 with new leaf values' | 100 | 1 | true || 5.5 | 200 + 'Replace list with 100 new nodes' | 100 | 101 | false || 9.0 | 200 + 'Replace list with 50 existing and 50 new' | 100 | 151 | true || 7.0 | 200 + 'Replace list of 100 nodes with 1' | 1 | 1 | false || 5.9 | 200 } def 'Update leaves for 100 data nodes.'() { @@ -125,8 +125,8 @@ class UpdatePerfTest extends CpsPerfTestBase { resourceMeter.stop() then: 'data leaves have expected values' assert 100 == countDataNodes('/openroadm-devices/openroadm-device[@status="fail"]') - and: 'update completes within expected time and a margin of 50%' - recordAndAssertResourceUsage('Update leaves for 100 data nodes', 0.14, resourceMeter.getTotalTimeInSeconds(), 120, resourceMeter.getTotalMemoryUsageInMB()) + and: 'update completes within expected time and a margin of 100%' + recordAndAssertResourceUsage('Update leaves for 100 data nodes', 0.22, resourceMeter.getTotalTimeInSeconds(), 120, resourceMeter.getTotalMemoryUsageInMB()) } def 'Clean up for CPS Update API.'() { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy index 21a9f19ea6..642f3f3809 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy @@ -39,18 +39,16 @@ class WritePerfTest extends CpsPerfTestBase { resourceMeter.start() cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, jsonData, OffsetDateTime.now()) resourceMeter.stop() - then: 'the operation takes less than #expectedDuration with a margin of 50%' - recordAndAssertResourceUsage("Writing ${totalNodes} devices", - expectedDuration, resourceMeter.getTotalTimeInSeconds(), - memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) + then: 'the operation takes less than #expectedDuration with a margin of 100%' + recordAndAssertResourceUsage("Writing ${totalNodes} devices", expectedDuration, resourceMeter.getTotalTimeInSeconds(), memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) cleanup: cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR) where: totalNodes || expectedDuration | memoryLimit 50 || 1.45 | 100 100 || 2.9 | 200 - 200 || 5.6 | 400 - 400 || 11.4 | 500 + 200 || 6.2 | 400 + 400 || 13.0 | 500 } def 'Writing bookstore data has exponential time.'() { @@ -64,7 +62,7 @@ class WritePerfTest extends CpsPerfTestBase { resourceMeter.start() cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, '/bookstore/categories[@code=1]', booksData, OffsetDateTime.now()) resourceMeter.stop() - then: 'the operation takes less than #expectedDuration with a margin of 50%' + then: 'the operation takes less than #expectedDuration with a margin of 100%' recordAndAssertResourceUsage("Writing ${totalBooks} books", expectedDuration, resourceMeter.totalTimeInSeconds, memoryLimit, resourceMeter.totalMemoryUsageInMB) @@ -72,8 +70,8 @@ class WritePerfTest extends CpsPerfTestBase { cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR) where: totalBooks || expectedDuration | memoryLimit - 800 || 0.26 | 50 - 1600 || 0.75 | 100 + 800 || 0.31 | 50 + 1600 || 0.8 | 100 3200 || 2.2 | 150 6400 || 6.9 | 200 } @@ -92,18 +90,16 @@ class WritePerfTest extends CpsPerfTestBase { resourceMeter.start() cpsDataService.saveListElements(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, '/openroadm-devices', jsonListData, OffsetDateTime.now(), ContentType.JSON) resourceMeter.stop() - then: 'the operation takes less than #expectedDuration with a margin of 50%' - recordAndAssertResourceUsage("Saving list of ${totalNodes} devices", - expectedDuration, resourceMeter.totalTimeInSeconds, - memoryLimit, resourceMeter.totalMemoryUsageInMB) + then: 'the operation takes less than #expectedDuration with a margin of 100%' + recordAndAssertResourceUsage("Saving list of ${totalNodes} devices", expectedDuration, resourceMeter.totalTimeInSeconds, memoryLimit, resourceMeter.totalMemoryUsageInMB) cleanup: cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR) where: totalNodes || expectedDuration | memoryLimit - 50 || 1.35 | 100 - 100 || 2.7 | 200 - 200 || 5.4 | 400 - 400 || 10.8 | 500 + 50 || 1.5 | 100 + 100 || 3.0 | 200 + 200 || 7.1 | 400 + 400 || 12.5 | 500 } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy index b4456e3b26..a7a1760077 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy @@ -97,7 +97,7 @@ class CmDataSubscriptionsPerfTest extends NcmpPerfTestBase { def resultAfter = objectUnderTest.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_ANCHOR, cpsPath, INCLUDE_ALL_DESCENDANTS) assert resultAfter.collect {it.leaves.subscribers.size()}.sum() == totalNumberOfEntries * (1 + numberOfCmDataSubscribers) and: 'update matching subscription within 15 seconds' - recordAndAssertResourceUsage("Update matching subscription", 6.2, durationInSeconds, 1000, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Update matching subscription", 10.5, durationInSeconds, 1000, resourceMeter.getTotalMemoryUsageInMB()) } def 'Worst case new subscription (200x10 new entries).'() { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy index 5f04519998..3216cc1eff 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy @@ -80,7 +80,7 @@ class CmHandleQueryPerfTest extends NcmpPerfTestBase { then: '100 data node have been found' assert count == 100 and: 'average performance is as expected' - recordAndAssertResourceUsage('Look up CM-handle by id', 0.25, resourceMeter.totalTimeInSeconds, 15, resourceMeter.totalMemoryUsageInMB) + recordAndAssertResourceUsage('Look up CM-handle by id', 0.58, resourceMeter.totalTimeInSeconds, 15, resourceMeter.totalMemoryUsageInMB) } def 'CM-handle is looked up by alternate id.'() { @@ -95,7 +95,7 @@ class CmHandleQueryPerfTest extends NcmpPerfTestBase { then: 'all alternate ids are resolved correctly' assert count == 100 and: 'average performance is as expected' - recordAndAssertResourceUsage('Look up CM-handle by alternate-id', 1.75, resourceMeter.totalTimeInSeconds, 15, resourceMeter.totalMemoryUsageInMB) + recordAndAssertResourceUsage('Look up CM-handle by alternate-id', 1.4, resourceMeter.totalTimeInSeconds, 15, resourceMeter.totalMemoryUsageInMB) } def 'A batch of CM-handles is looked up by alternate id.'() { @@ -125,7 +125,7 @@ class CmHandleQueryPerfTest extends NcmpPerfTestBase { assert count == TOTAL_CM_HANDLES * 100 then: 'average performance is as expected' def averageResponseTime = resourceMeter.totalTimeInSeconds / 100 - recordAndAssertResourceUsage('Look up CM-handles by module-set-tag', 0.26, averageResponseTime, 500, resourceMeter.totalMemoryUsageInMB) + recordAndAssertResourceUsage('Look up CM-handles by module-set-tag', 0.25, averageResponseTime, 500, resourceMeter.totalMemoryUsageInMB) } }