From 09086b5a7ce9a7467b42c6a86374f22318aaa4cd Mon Sep 17 00:00:00 2001 From: danielhanrahan Date: Mon, 13 Nov 2023 18:17:21 +0000 Subject: [PATCH] Adjust performance test timings New limits for time and memory are based on analyzing 40 previous performance job runs. Test limits are chosen so that individual test cases would pass 95% of the the time. Issue-ID: CPS-1884 Signed-off-by: danielhanrahan Change-Id: Iadca27eb1f9482016759070e5afdc70c53f9d043 --- .../performance/base/CpsPerfTestBase.groovy | 4 +-- .../performance/base/PerfTestBase.groovy | 2 +- .../cps/CpsDataServiceLimitsPerfTest.groovy | 4 +-- .../performance/cps/DeletePerfTest.groovy | 24 ++++++++--------- .../integration/performance/cps/GetPerfTest.groovy | 16 +++++------ .../performance/cps/QueryPerfTest.groovy | 30 ++++++++++----------- .../performance/cps/UpdatePerfTest.groovy | 8 +++--- .../performance/cps/WritePerfTest.groovy | 31 +++++++++------------- .../ncmp/CmDataSubscriptionsPerfTest.groovy | 8 +++--- .../performance/ncmp/CmHandleQueryPerfTest.groovy | 4 +-- .../org/onap/cps/integration/ResourceMeter.java | 4 +-- 11 files changed, 64 insertions(+), 71 deletions(-) diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy index 8cdbdc595..d8012ec6d 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy @@ -67,7 +67,7 @@ class CpsPerfTestBase extends PerfTestBase { addAnchorsWithData(OPENROADM_ANCHORS, CPS_PERFORMANCE_TEST_DATASPACE, LARGE_SCHEMA_SET, 'openroadm', data) resourceMeter.stop() def durationInSeconds = resourceMeter.getTotalTimeInSeconds() - recordAndAssertResourceUsage('Creating openroadm anchors with large data tree', 200, durationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Creating openroadm anchors with large data tree', 200, durationInSeconds, 600, resourceMeter.getTotalMemoryUsageInMB()) } def generateOpenRoadData(numberOfNodes) { @@ -87,7 +87,7 @@ class CpsPerfTestBase extends PerfTestBase { then: 'memory used is within #peakMemoryUsage' assert resourceMeter.getTotalMemoryUsageInMB() <= 30 and: 'all data is read within expected time' - recordAndAssertResourceUsage("Warming database", 200, durationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Warming database", 200, durationInSeconds, 600, resourceMeter.getTotalMemoryUsageInMB()) } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy index b0a105c7f..a96d7f64b 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy @@ -55,7 +55,7 @@ abstract class PerfTestBase extends CpsIntegrationSpecBase { abstract def createInitialData() def recordAndAssertResourceUsage(String shortTitle, double thresholdInSec, double recordedTimeInSec, memoryLimit, memoryUsageInMB) { - def pass = recordedTimeInSec <= thresholdInSec + def pass = recordedTimeInSec <= thresholdInSec && memoryUsageInMB <= memoryLimit if (shortTitle.length() > 40) { shortTitle = shortTitle.substring(0, 40) } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy index 6a3d4434b..ce66cb08a 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy @@ -47,7 +47,7 @@ class CpsDataServiceLimitsPerfTest extends CpsPerfTestBase { resourceMeter.stop() def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'the operation completes within 25 seconds' - recordAndAssertResourceUsage("Creating 33,000 books", 25, durationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Creating 33,000 books", 25, durationInSeconds, 150, resourceMeter.getTotalMemoryUsageInMB()) } def 'Get data nodes from multiple xpaths 32K (2^15) limit exceeded.'() { @@ -88,7 +88,7 @@ class CpsDataServiceLimitsPerfTest extends CpsPerfTestBase { resourceMeter.stop() def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'test data is deleted in 1 second' - recordAndAssertResourceUsage("Deleting test data", 1, durationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Deleting test data", 1, durationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB()) } def countDataNodes() { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy index 0dcd995a7..818c300a5 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy @@ -40,7 +40,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def setupDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'setup duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete test setup', 200, setupDurationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete test setup', 200, setupDurationInSeconds, 800, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete 100 container nodes'() { @@ -56,7 +56,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete 100 containers', 2, deleteDurationInSeconds, 30, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete 100 containers', 2.5, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch delete 100 container nodes'() { @@ -70,7 +70,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Batch delete 100 containers', 0.5, deleteDurationInSeconds, 5, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Batch delete 100 containers', 0.6, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete 100 list elements'() { @@ -86,7 +86,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete 100 lists elements', 2, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete 100 lists elements', 2.5, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch delete 100 list elements'() { @@ -100,7 +100,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Batch delete 100 lists elements', 0.5, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Batch delete 100 lists elements', 0.6, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete 100 whole lists'() { @@ -116,7 +116,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete 100 whole lists', 5, deleteDurationInSeconds, 30, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete 100 whole lists', 6, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch delete 100 whole lists'() { @@ -130,7 +130,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Batch delete 100 whole lists', 4, deleteDurationInSeconds, 5, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Batch delete 100 whole lists', 5, deleteDurationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete 1 large data node'() { @@ -140,7 +140,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete one large node', 2, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete one large node', 2, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete root node with many descendants'() { @@ -150,7 +150,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete root node', 2, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete root node', 2, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete data nodes for an anchor'() { @@ -160,7 +160,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete data nodes for anchor', 2, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete data nodes for anchor', 2, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch delete 100 non-existing nodes'() { @@ -174,7 +174,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Batch delete 100 non-existing', 7, deleteDurationInSeconds, 5, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Batch delete 100 non-existing', 7, deleteDurationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB()) } def 'Clean up test data'() { @@ -186,7 +186,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete test cleanup', 10, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete test cleanup', 10, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB()) } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy index 95cf260f2..8a228a353 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy @@ -44,9 +44,9 @@ class GetPerfTest extends CpsPerfTestBase { recordAndAssertResourceUsage("Read datatrees with ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' scenario | fetchDescendantsOption || durationLimit | memoryLimit | expectedNumberOfDataNodes - 'no descendants' | OMIT_DESCENDANTS || 0.01 | 5 | 1 - 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.05 | 10 | 1 + OPENROADM_DEVICES_PER_ANCHOR - 'all descendants' | INCLUDE_ALL_DESCENDANTS || 2 | 200 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'no descendants' | OMIT_DESCENDANTS || 0.02 | 1 | 1 + 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.06 | 5 | 1 + OPENROADM_DEVICES_PER_ANCHOR + 'all descendants' | INCLUDE_ALL_DESCENDANTS || 2.5 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } def 'Read data trees for multiple xpaths'() { @@ -60,7 +60,7 @@ class GetPerfTest extends CpsPerfTestBase { then: 'requested nodes and their descendants are returned' assert countDataNodesInTree(result) == OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE and: 'all data is read within expected time and memory used is within limit' - recordAndAssertResourceUsage("Read datatrees for multiple xpaths", 3 , durationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Read datatrees for multiple xpaths", 4 , durationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB()) } def 'Read for multiple xpaths to non-existing datanodes'() { @@ -74,7 +74,7 @@ class GetPerfTest extends CpsPerfTestBase { then: 'no data is returned' assert result.isEmpty() and: 'the operation completes within within expected time' - recordAndAssertResourceUsage("Read non-existing xpaths", 0.01, durationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Read non-existing xpaths", 0.02, durationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) } def 'Read complete data trees using #scenario.'() { @@ -88,9 +88,9 @@ class GetPerfTest extends CpsPerfTestBase { recordAndAssertResourceUsage("Read datatrees using ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following xpaths are used' scenario | xpath || durationLimit | memoryLimit | expectedNumberOfDataNodes - 'openroadm root' | '/' || 2 | 200 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE - 'openroadm top element' | '/openroadm-devices' || 2 | 200 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE - 'openroadm whole list' | '/openroadm-devices/openroadm-device' || 3 | 200 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'openroadm root' | '/' || 2 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'openroadm top element' | '/openroadm-devices' || 2 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'openroadm whole list' | '/openroadm-devices/openroadm-device' || 3 | 250 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy index 5cf495563..0ae018d46 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy @@ -45,10 +45,11 @@ class QueryPerfTest extends CpsPerfTestBase { recordAndAssertResourceUsage("Query 1 anchor ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' scenario | cpsPath || durationLimit | memoryLimit | expectedNumberOfDataNodes - 'top element' | '/openroadm-devices' || 2 | 300 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 - 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || 3 | 200 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE - 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 2 | 200 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 - 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 2 | 300 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 + 'top element' | '/openroadm-devices' || 2.5 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 + 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || 2.5 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 2.5 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 + 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 2.5 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 + 'non-existing data' | '/path/to/non-existing/node[@id="1"]' || 0.1 | 1 | 0 } def 'Query complete data trees across all anchors with #scenario.'() { @@ -63,11 +64,10 @@ class QueryPerfTest extends CpsPerfTestBase { recordAndAssertResourceUsage("Query across anchors ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' scenario | cpspath || durationLimit | memoryLimit | expectedNumberOfDataNodes - 'top element' | '/openroadm-devices' || 6 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) - 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || 6 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE) - 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 6 | 800 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) - 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 6 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) - 'non-existing data' | '/path/to/non-existing/node[@id="1"]' || 0.1 | 3 | 0 + 'top element' | '/openroadm-devices' || 7 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) + 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || 7 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE) + 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 7 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) + 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 7 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) } def 'Query with leaf condition and #scenario.'() { @@ -82,9 +82,9 @@ class QueryPerfTest extends CpsPerfTestBase { recordAndAssertResourceUsage("Query with ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' scenario | fetchDescendantsOption || durationLimit | memoryLimit | expectedNumberOfDataNodes - 'no descendants' | OMIT_DESCENDANTS || 0.1 | 30 | OPENROADM_DEVICES_PER_ANCHOR - 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.15 | 30 | OPENROADM_DEVICES_PER_ANCHOR * 2 - 'all descendants' | INCLUDE_ALL_DESCENDANTS || 2 | 200 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'no descendants' | OMIT_DESCENDANTS || 0.1 | 6 | OPENROADM_DEVICES_PER_ANCHOR + 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.2 | 12 | OPENROADM_DEVICES_PER_ANCHOR * 2 + 'all descendants' | INCLUDE_ALL_DESCENDANTS || 2.5 | 200 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } def 'Query ancestors with #scenario.'() { @@ -99,9 +99,9 @@ class QueryPerfTest extends CpsPerfTestBase { recordAndAssertResourceUsage("Query ancestors with ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' scenario | fetchDescendantsOption || durationLimit | memoryLimit | expectedNumberOfDataNodes - 'no descendants' | OMIT_DESCENDANTS || 0.1 | 20 | 1 - 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.1 | 20 | 1 + OPENROADM_DEVICES_PER_ANCHOR - 'all descendants' | INCLUDE_ALL_DESCENDANTS || 2 | 200 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'no descendants' | OMIT_DESCENDANTS || 0.1 | 3 | 1 + 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.2 | 8 | 1 + OPENROADM_DEVICES_PER_ANCHOR + 'all descendants' | INCLUDE_ALL_DESCENDANTS || 2.5 | 400 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy index 151492dc9..8118010b4 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy @@ -41,7 +41,7 @@ class UpdatePerfTest extends CpsPerfTestBase { resourceMeter.stop() def updateDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'update completes within expected time and memory used is within limit' - recordAndAssertResourceUsage('Update 1 data node', 0.6, updateDurationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Update 1 data node', 0.6, updateDurationInSeconds, 100, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch update 100 data nodes with descendants'() { @@ -57,7 +57,7 @@ class UpdatePerfTest extends CpsPerfTestBase { resourceMeter.stop() def updateDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'update completes within expected time and memory used is within limit' - recordAndAssertResourceUsage('Update 100 data nodes', 30, updateDurationInSeconds, 800, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Update 100 data nodes', 40, updateDurationInSeconds, 800, resourceMeter.getTotalMemoryUsageInMB()) } def 'Update leaves for 1 data node (twice)'() { @@ -71,7 +71,7 @@ class UpdatePerfTest extends CpsPerfTestBase { resourceMeter.stop() def updateDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'update completes within expected time and memory used is within limit' - recordAndAssertResourceUsage('Update leaves for 1 data node', 0.5, updateDurationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Update leaves for 1 data node', 0.7, updateDurationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch update leaves for 100 data nodes (twice)'() { @@ -85,7 +85,7 @@ class UpdatePerfTest extends CpsPerfTestBase { resourceMeter.stop() def updateDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'update completes within expected time and memory used is within limit' - recordAndAssertResourceUsage('Batch update leaves for 100 data nodes', 1, updateDurationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Batch update leaves for 100 data nodes', 1, updateDurationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB()) } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy index 177cd9fc5..0c7107a56 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy @@ -36,19 +36,16 @@ class WritePerfTest extends CpsPerfTestBase { resourceMeter.stop() def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'the operation takes less than #expectedDuration and memory used is within limit' - recordAndAssertResourceUsage("Writing ${totalNodes} devices", expectedDurationInSeconds, durationInSeconds, 400, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Writing ${totalNodes} devices", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) cleanup: cpsDataService.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor', OffsetDateTime.now()) cpsAdminService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor') where: - totalNodes || expectedDurationInSeconds - 50 || 3 - 100 || 5 - 200 || 10 - 400 || 20 -// 800 || 40 -// 1600 || 80 -// 3200 || 160 + totalNodes || expectedDuration | memoryLimit + 50 || 4 | 100 + 100 || 7 | 200 + 200 || 14 | 400 + 400 || 28 | 500 } def 'Writing bookstore data has exponential time.'() { @@ -64,20 +61,16 @@ class WritePerfTest extends CpsPerfTestBase { resourceMeter.stop() def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'the operation takes less than #expectedDuration and memory used is within limit' - recordAndAssertResourceUsage("Writing ${totalBooks} books", expectedDuration, durationInSeconds, 400, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Writing ${totalBooks} books", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) cleanup: cpsDataService.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor', OffsetDateTime.now()) cpsAdminService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor') where: - totalBooks || expectedDuration - 400 || 0.2 - 800 || 0.5 - 1600 || 1 - 3200 || 3 - 6400 || 10 -// 12800 || 30 -// 25600 || 120 -// 51200 || 600 + totalBooks || expectedDuration | memoryLimit + 800 || 1 | 50 + 1600 || 2 | 100 + 3200 || 6 | 150 + 6400 || 18 | 200 } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy index 896217a2a..579394be8 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy @@ -52,7 +52,7 @@ class CmDataSubscriptionsPerfTest extends NcmpPerfTestBase { matches.size() == numberOfFiltersPerCmHandle * numberOfCmHandlesPerCmDataSubscription and: 'query all subscribers within 1 second' def durationInSeconds = resourceMeter.getTotalTimeInSeconds() - recordAndAssertResourceUsage("Query all subscribers", 1, durationInSeconds, 400, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Query all subscribers", 1.2, durationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB()) } def 'Worst case subscription update (200x10 matching entries).'() { @@ -94,8 +94,8 @@ class CmDataSubscriptionsPerfTest extends NcmpPerfTestBase { then: 'a subscriber has been added to each filter entry' def resultAfter = objectUnderTest.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_ANCHOR, cpsPath, INCLUDE_ALL_DESCENDANTS) assert resultAfter.collect {it.leaves.subscribers.size()}.sum() == totalNumberOfEntries * (1 + numberOfCmDataSubscribers) - and: 'update matching subscription within 8 seconds' - recordAndAssertResourceUsage("Update matching subscription", 8, durationInSeconds, 400, resourceMeter.getTotalMemoryUsageInMB()) + and: 'update matching subscription within 15 seconds' + recordAndAssertResourceUsage("Update matching subscription", 15, durationInSeconds, 1000, resourceMeter.getTotalMemoryUsageInMB()) } def 'Worst case new subscription (200x10 new entries).'() { @@ -109,7 +109,7 @@ class CmDataSubscriptionsPerfTest extends NcmpPerfTestBase { resourceMeter.stop() def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'insert new subscription with 1 second' - recordAndAssertResourceUsage("Insert new subscription", 1, durationInSeconds, 400,resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Insert new subscription", 2, durationInSeconds, 100, resourceMeter.getTotalMemoryUsageInMB()) } def querySubscriptionsByIteration(Collection allSubscriptionsAsDataNodes, targetSubscriptionSequenceNumber) { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy index becd7e39c..a5a6acb7a 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy @@ -45,8 +45,8 @@ class CmHandleQueryPerfTest extends NcmpPerfTestBase { def result = cpsDataService.getDataNodesForMultipleXpaths(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR, xpaths, INCLUDE_ALL_DESCENDANTS) resourceMeter.stop() def durationInSeconds = resourceMeter.getTotalTimeInSeconds() - then: 'the required operations are performed within 1200 ms' - recordAndAssertResourceUsage("CpsPath Registry attributes Query", 0.25, durationInSeconds, 150, resourceMeter.getTotalMemoryUsageInMB()) + then: 'the required operations are performed within required time' + recordAndAssertResourceUsage("CpsPath Registry attributes Query", 0.4, durationInSeconds, 50, resourceMeter.getTotalMemoryUsageInMB()) and: 'all but 1 (other node) are returned' result.size() == 999 and: 'the tree contains all the expected descendants too' diff --git a/integration-test/src/test/java/org/onap/cps/integration/ResourceMeter.java b/integration-test/src/test/java/org/onap/cps/integration/ResourceMeter.java index 1e420013d..c7d96c4c2 100644 --- a/integration-test/src/test/java/org/onap/cps/integration/ResourceMeter.java +++ b/integration-test/src/test/java/org/onap/cps/integration/ResourceMeter.java @@ -48,8 +48,8 @@ public class ResourceMeter { } /** - * Get the total time in milliseconds. - * @return total time in milliseconds + * Get the total time in seconds. + * @return total time in seconds */ public double getTotalTimeInSeconds() { return stopWatch.getTotalTimeSeconds(); -- 2.16.6