From: ToineSiebelink Date: Tue, 20 May 2025 15:43:49 +0000 (+0100) Subject: Make Groovy Performance Test Stable X-Git-Tag: 3.6.3~9^2 X-Git-Url: https://gerrit.onap.org/r/gitweb?a=commitdiff_plain;h=d2d72b299762ccae0b7cfe8a4af6bccffc1b8baa;p=cps.git Make Groovy Performance Test Stable - Removed average calculation where this lead to very small duration and therefore seemingly more variation - Tweak a few expected duration to get closer to actual average - Remove memory assertion from resource meter test since they are unreliable and cause the odd error - Corrected wording on description related to limits Issue-ID:CPS-2743 Change-Id: Id30bdbe7074297c32a236322fc66ee698fdd8e2b Signed-off-by: ToineSiebelink --- diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/ResourceMeterAccuracyTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/ResourceMeterAccuracyTest.groovy new file mode 100644 index 0000000000..0bb4720626 --- /dev/null +++ b/integration-test/src/test/groovy/org/onap/cps/integration/ResourceMeterAccuracyTest.groovy @@ -0,0 +1,41 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2023-2025 OpenInfra Foundation Europe. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.integration + +import spock.lang.Specification + +import java.util.concurrent.TimeUnit + +class ResourceMeterAccuracyTest extends Specification { + + def objectUnderTest = new ResourceMeter() + + def 'Resource meter duration measurement.'() { + when: 'we measure how long a known operation takes' + objectUnderTest.start() + TimeUnit.SECONDS.sleep(2) + objectUnderTest.stop() + then: 'the resource meter reports a duration within 20ms of the expected duration' + assert objectUnderTest.getTotalTimeInSeconds() >= 1.98 + assert objectUnderTest.getTotalTimeInSeconds() <= 2.02 + } + +} diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/ResourceMeterPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/ResourceMeterPerfTest.groovy deleted file mode 100644 index c42bfd7be6..0000000000 --- a/integration-test/src/test/groovy/org/onap/cps/integration/ResourceMeterPerfTest.groovy +++ /dev/null @@ -1,83 +0,0 @@ -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2023 Nordix Foundation - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the 'License'); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an 'AS IS' BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.onap.cps.integration - -import java.util.concurrent.TimeUnit -import spock.lang.Specification - -class ResourceMeterPerfTest extends Specification { - - final int MEGABYTE = 1_000_000 - - def resourceMeter = new ResourceMeter() - - def 'ResourceMeter accurately measures duration'() { - when: 'we measure how long a known operation takes' - resourceMeter.start() - TimeUnit.SECONDS.sleep(2) - resourceMeter.stop() - then: 'ResourceMeter reports a duration within 10ms of the expected duration' - assert resourceMeter.getTotalTimeInSeconds() >= 2 - assert resourceMeter.getTotalTimeInSeconds() <= 2.01 - } - - def 'ResourceMeter reports memory usage when allocating a large byte array'() { - when: 'the resource meter is started' - resourceMeter.start() - and: 'some memory is allocated' - byte[] array = new byte[50 * MEGABYTE] - and: 'the resource meter is stopped' - resourceMeter.stop() - then: 'the reported memory usage is close to the amount of memory allocated' - assert resourceMeter.getTotalMemoryUsageInMB() >= 50 - assert resourceMeter.getTotalMemoryUsageInMB() <= 55 - } - - def 'ResourceMeter measures PEAK memory usage when garbage collector runs'() { - when: 'the resource meter is started' - resourceMeter.start() - and: 'some memory is allocated' - byte[] array = new byte[50 * MEGABYTE] - and: 'the memory is garbage collected' - array = null - ResourceMeter.performGcAndWait() - and: 'the resource meter is stopped' - resourceMeter.stop() - then: 'the reported memory usage is close to the peak amount of memory allocated' - assert resourceMeter.getTotalMemoryUsageInMB() >= 50 - assert resourceMeter.getTotalMemoryUsageInMB() <= 55 - } - - def 'ResourceMeter measures memory increase only during measurement'() { - given: '50 megabytes is allocated before measurement' - byte[] arrayBefore = new byte[50 * MEGABYTE] - when: 'memory is allocated during measurement' - resourceMeter.start() - byte[] arrayDuring = new byte[40 * MEGABYTE] - resourceMeter.stop() - and: '50 megabytes is allocated after measurement' - byte[] arrayAfter = new byte[50 * MEGABYTE] - then: 'the reported memory usage is close to the amount allocated DURING measurement' - assert resourceMeter.getTotalMemoryUsageInMB() >= 40 - assert resourceMeter.getTotalMemoryUsageInMB() <= 45 - } - -} diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy index f98f1c6f91..d869ba775e 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy @@ -26,9 +26,9 @@ abstract class PerfTestBase extends CpsIntegrationSpecBase { static def LARGE_SCHEMA_SET = 'largeSchemaSet' static def PERFORMANCE_RECORD = [] - static def DEFAULT_TIME_MARGIN = 1.5 // Allow 50% margin - static def VERY_FAST_TEST_THRESHOLD = 0.01 // Defintion of a very vast test (hard to measure) - static def DEFAULT_TIME_MARGIN_FOR_VERY_FAST_TEST = 3 // Allow 200% margin on very fast test (accuracy is an issue) + static def DEFAULT_TIME_LIMIT_FACTOR = 1.5 // Allow 50% margin on top of expected (average) value + static def VERY_FAST_TEST_THRESHOLD = 0.01 // Definition of a very vast test (hard to measure) + static def DEFAULT_TIME_LIMIT_FACTOR_FOR_VERY_FAST_TEST = 3 // Allow 200% margin on very fast test (accuracy is an issue) def cleanupSpec() { println('##################################################################################################') @@ -56,22 +56,22 @@ abstract class PerfTestBase extends CpsIntegrationSpecBase { abstract def createInitialData() - def recordAndAssertResourceUsage(String shortTitle, double expectedTimeInSec, double recordedTimeInSec, memoryLimit, memoryUsageInMB, double timeMargin) { - if (expectedTimeInSec <= VERY_FAST_TEST_THRESHOLD) { - timeMargin = DEFAULT_TIME_MARGIN_FOR_VERY_FAST_TEST + def recordAndAssertResourceUsage(shortTitle, expectedAverageTimeInSec, recordedTimeInSec, memoryLimitCurrentlyNotAsserted, memoryUsageInMB, timeLimitFactor) { + if (expectedAverageTimeInSec <= VERY_FAST_TEST_THRESHOLD) { + timeLimitFactor = DEFAULT_TIME_LIMIT_FACTOR_FOR_VERY_FAST_TEST } - def testPassed = recordedTimeInSec <= timeMargin * expectedTimeInSec + def testPassed = recordedTimeInSec <= timeLimitFactor * expectedAverageTimeInSec if (shortTitle.length() > 40) { shortTitle = shortTitle.substring(0, 40) } - def record = String.format('%2d.%-40s limit %8.3f took %8.3f sec %,8.2f MB used ', PERFORMANCE_RECORD.size() + 1, shortTitle, expectedTimeInSec, recordedTimeInSec, memoryUsageInMB) + def record = String.format('%2d.%-40s limit %8.3f took %8.3f sec %,8.2f MB used ', PERFORMANCE_RECORD.size() + 1, shortTitle, expectedAverageTimeInSec, recordedTimeInSec, memoryUsageInMB) record += testPassed ? 'PASS' : 'FAIL' PERFORMANCE_RECORD.add(record) - assert recordedTimeInSec <= timeMargin * expectedTimeInSec + assert recordedTimeInSec <= timeLimitFactor * expectedAverageTimeInSec return true } def recordAndAssertResourceUsage(String shortTitle, double thresholdInSec, double recordedTimeInSec, memoryLimit, memoryUsageInMB) { - recordAndAssertResourceUsage(shortTitle, thresholdInSec, recordedTimeInSec, memoryLimit, memoryUsageInMB, DEFAULT_TIME_MARGIN) + recordAndAssertResourceUsage(shortTitle, thresholdInSec, recordedTimeInSec, memoryLimit, memoryUsageInMB, DEFAULT_TIME_LIMIT_FACTOR) } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsModuleServicePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsModuleServicePerfTest.groovy index d8553419ce..9c0187d5b7 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsModuleServicePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsModuleServicePerfTest.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2022-2023 Nordix Foundation + * Copyright (C) 2022-2025 OpenInfra Foundation Europe. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,8 +21,8 @@ package org.onap.cps.integration.performance.cps import org.onap.cps.api.CpsModuleService -import org.onap.cps.integration.performance.base.CpsPerfTestBase import org.onap.cps.api.model.ModuleReference +import org.onap.cps.integration.performance.base.CpsPerfTestBase import org.springframework.util.StopWatch import java.util.concurrent.ThreadLocalRandom @@ -68,7 +68,7 @@ class CpsModuleServicePerfTest extends CpsPerfTestBase { stopWatch.stop() assert newModuleReferences.size() > 0 && newModuleReferences.size() < 300 } - assert stopWatch.getTotalTimeMillis() < 6000 + assert stopWatch.getTotalTimeMillis() < 10_000 } def createModuleReferencesWithRandomMatchingExistingModuleReferences() { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy index 94e809eda2..d93ea4b309 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy @@ -40,7 +40,7 @@ class DeletePerfTest extends CpsPerfTestBase { addAnchorsWithData(10, CPS_PERFORMANCE_TEST_DATASPACE, LARGE_SCHEMA_SET, 'delete', data, ContentType.JSON) resourceMeter.stop() def setupDurationInSeconds = resourceMeter.getTotalTimeInSeconds() - then: 'setup duration is within expected time and memory used is within limit' + then: 'setup duration is below accepted margin of the expected average' recordAndAssertResourceUsage('Delete test setup', 103, setupDurationInSeconds, 800, resourceMeter.getTotalMemoryUsageInMB()) } @@ -56,8 +56,8 @@ class DeletePerfTest extends CpsPerfTestBase { } resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() - then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete 100 containers', 1.5, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB()) + then: 'delete duration is below accepted margin of the expected average' + recordAndAssertResourceUsage('Delete 100 containers', 1.4, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch delete 100 container nodes'() { @@ -70,7 +70,7 @@ class DeletePerfTest extends CpsPerfTestBase { objectUnderTest.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'delete2', xpathsToDelete, OffsetDateTime.now()) resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() - then: 'delete duration is within expected time and memory used is within limit' + then: 'delete duration is below accepted margin of the expected average' recordAndAssertResourceUsage('Batch delete 100 containers', 0.65, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) } @@ -86,7 +86,7 @@ class DeletePerfTest extends CpsPerfTestBase { } resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() - then: 'delete duration is within expected time and memory used is within limit' + then: 'delete duration is below accepted margin of the expected average' recordAndAssertResourceUsage('Delete 100 lists elements', 1.5, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB()) } @@ -100,7 +100,7 @@ class DeletePerfTest extends CpsPerfTestBase { objectUnderTest.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'delete4', xpathsToDelete, OffsetDateTime.now()) resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() - then: 'delete duration is within expected time and memory used is within limit' + then: 'delete duration is below accepted margin of the expected average' recordAndAssertResourceUsage('Batch delete 100 lists elements', 0.7, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) } @@ -116,8 +116,8 @@ class DeletePerfTest extends CpsPerfTestBase { } resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() - then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete 100 whole lists', 3.2, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB(), 3) + then: 'delete duration is below accepted margin (4x) of the expected average' + recordAndAssertResourceUsage('Delete 100 whole lists', 3.2, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB(), 4) } def 'Batch delete 100 whole lists'() { @@ -130,7 +130,7 @@ class DeletePerfTest extends CpsPerfTestBase { objectUnderTest.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'delete6', xpathsToDelete, OffsetDateTime.now()) resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() - then: 'delete duration is within expected time and memory used is within limit' + then: 'delete duration is below accepted margin (3x) of the expected average' recordAndAssertResourceUsage('Batch delete 100 whole lists', 3.4, deleteDurationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB(), 3) } @@ -140,7 +140,7 @@ class DeletePerfTest extends CpsPerfTestBase { objectUnderTest.deleteDataNode(CPS_PERFORMANCE_TEST_DATASPACE, 'delete7', '/openroadm-devices', OffsetDateTime.now()) resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() - then: 'delete duration is within expected time and memory used is within limit' + then: 'delete duration is below accepted margin of the expected average' recordAndAssertResourceUsage('Delete one large node', 2, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB()) } @@ -150,7 +150,7 @@ class DeletePerfTest extends CpsPerfTestBase { objectUnderTest.deleteDataNode(CPS_PERFORMANCE_TEST_DATASPACE, 'delete8', '/', OffsetDateTime.now()) resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() - then: 'delete duration is within expected time and memory used is within limit' + then: 'delete duration is below accepted margin of the expected average' recordAndAssertResourceUsage('Delete root node', 2, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB()) } @@ -160,7 +160,7 @@ class DeletePerfTest extends CpsPerfTestBase { objectUnderTest.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'delete9', OffsetDateTime.now()) resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() - then: 'delete duration is within expected time and memory used is within limit' + then: 'delete duration is below accepted margin of the expected average' recordAndAssertResourceUsage('Delete data nodes for anchor', 2, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB()) } @@ -174,7 +174,7 @@ class DeletePerfTest extends CpsPerfTestBase { } catch (DataNodeNotFoundException ignored) {} resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() - then: 'delete duration is within expected time and memory used is within limit' + then: 'delete duration is below accepted margin of the expected average' recordAndAssertResourceUsage('Batch delete 100 non-existing', 2, deleteDurationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB()) } @@ -186,7 +186,7 @@ class DeletePerfTest extends CpsPerfTestBase { cpsAnchorService.deleteAnchors(CPS_PERFORMANCE_TEST_DATASPACE, anchorNames) resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() - then: 'delete duration is within expected time and memory used is within limit' + then: 'delete duration is below accepted margin of the expected average' recordAndAssertResourceUsage('Delete test cleanup', 11.09, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB()) } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy index ba2c9cab2f..e42792e653 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy @@ -40,13 +40,13 @@ class GetPerfTest extends CpsPerfTestBase { resourceMeter.stop() assert countDataNodesInTree(result) == expectedNumberOfDataNodes def durationInSeconds = resourceMeter.getTotalTimeInSeconds() - then: 'all data is read within #durationLimit ms and memory used is within limit' - recordAndAssertResourceUsage("Read datatrees with ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) + then: 'all data is read within #expectedDuration seconds and a margin of 50%' + recordAndAssertResourceUsage("Read datatrees with ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' - scenario | fetchDescendantsOption || durationLimit | memoryLimit | expectedNumberOfDataNodes - 'no descendants' | OMIT_DESCENDANTS || 0.01 | 1 | 1 - 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.03 | 5 | 1 + OPENROADM_DEVICES_PER_ANCHOR - 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.1 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + scenario | fetchDescendantsOption || expectedDuration | memoryLimit | expectedNumberOfDataNodes + 'no descendants' | OMIT_DESCENDANTS || 0.01 | 1 | 1 + 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.03 | 5 | 1 + OPENROADM_DEVICES_PER_ANCHOR + 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.1 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } def 'Read data trees for multiple xpaths'() { @@ -59,8 +59,8 @@ class GetPerfTest extends CpsPerfTestBase { def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'requested nodes and their descendants are returned' assert countDataNodesInTree(result) == OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE - and: 'all data is read within expected time and memory used is within limit' - recordAndAssertResourceUsage("Read datatrees for multiple xpaths", 2.1, durationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB(),3) + and: 'all data is read within expected time and a margin of 200%' + recordAndAssertResourceUsage("Read datatrees for multiple xpaths", 1.1, durationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB(),3) } def 'Read for multiple xpaths to non-existing datanodes'() { @@ -84,13 +84,13 @@ class GetPerfTest extends CpsPerfTestBase { assert countDataNodesInTree(result) == expectedNumberOfDataNodes resourceMeter.stop() def durationInSeconds = resourceMeter.getTotalTimeInSeconds() - then: 'all data is read within expected time and memory used is within limit' - recordAndAssertResourceUsage("Read datatrees using ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) + then: 'all data is read within expected time and a margin of 50%' + recordAndAssertResourceUsage("Read datatrees using ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following xpaths are used' - scenario | xpath || durationLimit | memoryLimit | expectedNumberOfDataNodes - 'openroadm root' | '/' || 1 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE - 'openroadm top element' | '/openroadm-devices' || 0.9 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE - 'openroadm whole list' | '/openroadm-devices/openroadm-device' || 1.06 | 250 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + scenario | xpath || expectedDuration | memoryLimit | expectedNumberOfDataNodes + 'openroadm root' | '/' || 1 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'openroadm top element' | '/openroadm-devices' || 0.9 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'openroadm whole list' | '/openroadm-devices/openroadm-device' || 1.06 | 250 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy index bb8143c8c1..2b104992dc 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy @@ -41,15 +41,15 @@ class QueryPerfTest extends CpsPerfTestBase { def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'the expected number of nodes is returned' assert countDataNodesInTree(result) == expectedNumberOfDataNodes - and: 'all data is read within #durationLimit seconds and memory used is within limit' - recordAndAssertResourceUsage("Query 1 anchor ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) + and: 'all data is read #expectedDuration seconds with a margin of 50%' + recordAndAssertResourceUsage("Query 1 anchor ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' - scenario | cpsPath || durationLimit | memoryLimit | expectedNumberOfDataNodes - 'top element' | '/openroadm-devices' || 0.89 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 - 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || 1.3 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE - 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 1.1 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 - 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 0.93 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 - 'non-existing data' | '/path/to/non-existing/node[@id="1"]' || 0.01 | 1 | 0 + scenario | cpsPath || expectedDuration | memoryLimit | expectedNumberOfDataNodes + 'top element' | '/openroadm-devices' || 0.89 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 + 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || 1.3 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 1.1 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 + 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 0.93 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 + 'non-existing data' | '/path/to/non-existing/node[@id="1"]' || 0.01 | 1 | 0 } def 'Query complete data trees across all anchors with #scenario.'() { @@ -60,14 +60,14 @@ class QueryPerfTest extends CpsPerfTestBase { def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'the expected number of nodes is returned' assert countDataNodesInTree(result) == expectedNumberOfDataNodes - and: 'all data is read within #durationLimit seconds and memory used is within limit' - recordAndAssertResourceUsage("Query across anchors ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) + and: 'all data is read #expectedDuration seconds with a margin of 50%' + recordAndAssertResourceUsage("Query across anchors ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' - scenario | cpspath || durationLimit | memoryLimit | expectedNumberOfDataNodes - 'top element' | '/openroadm-devices' || 3.0 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) - 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || 3.3 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE) - 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 3.2 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) - 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 3.0 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) + scenario | cpspath || expectedDuration | memoryLimit | expectedNumberOfDataNodes + 'top element' | '/openroadm-devices' || 3.0 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) + 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || 3.3 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE) + 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 3.2 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) + 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 3.0 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) } def 'Query with leaf condition and #scenario.'() { @@ -78,13 +78,13 @@ class QueryPerfTest extends CpsPerfTestBase { def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'the expected number of nodes is returned' assert countDataNodesInTree(result) == expectedNumberOfDataNodes - and: 'all data is read within #durationLimit seconds and memory used is within limit' - recordAndAssertResourceUsage("Query with ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) + and: 'all data is read #expectedDuration seconds with a margin of 50%' + recordAndAssertResourceUsage("Query with ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' - scenario | fetchDescendantsOption || durationLimit | memoryLimit | expectedNumberOfDataNodes - 'no descendants' | OMIT_DESCENDANTS || 0.08 | 6 | OPENROADM_DEVICES_PER_ANCHOR - 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.09 | 12 | OPENROADM_DEVICES_PER_ANCHOR * 2 - 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.05 | 200 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + scenario | fetchDescendantsOption || expectedDuration | memoryLimit | expectedNumberOfDataNodes + 'no descendants' | OMIT_DESCENDANTS || 0.08 | 6 | OPENROADM_DEVICES_PER_ANCHOR + 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.09 | 12 | OPENROADM_DEVICES_PER_ANCHOR * 2 + 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.05 | 200 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } def 'Query ancestors with #scenario.'() { @@ -95,13 +95,13 @@ class QueryPerfTest extends CpsPerfTestBase { def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'the expected number of nodes is returned' assert countDataNodesInTree(result) == expectedNumberOfDataNodes - and: 'all data is read within #durationLimit seconds and memory used is within limit' - recordAndAssertResourceUsage("Query ancestors with ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) + and: 'all data is read #expectedDuration seconds with a margin of 50%' + recordAndAssertResourceUsage("Query ancestors with ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' - scenario | fetchDescendantsOption || durationLimit | memoryLimit | expectedNumberOfDataNodes - 'no descendants' | OMIT_DESCENDANTS || 0.06 | 3 | 1 - 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.07 | 8 | 1 + OPENROADM_DEVICES_PER_ANCHOR - 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.0 | 400 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + scenario | fetchDescendantsOption || expectedDuration | memoryLimit | expectedNumberOfDataNodes + 'no descendants' | OMIT_DESCENDANTS || 0.06 | 3 | 1 + 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.07 | 8 | 1 + OPENROADM_DEVICES_PER_ANCHOR + 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.0 | 400 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } def 'Query data leaf with #scenario.'() { @@ -112,13 +112,13 @@ class QueryPerfTest extends CpsPerfTestBase { def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'the expected number of results is returned' assert result.size() == expectedNumberOfValues - and: 'all data is read within #durationLimit seconds and memory used is within limit' - recordAndAssertResourceUsage("Query data leaf ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) + and: 'all data is read #expectedDuration seconds with a margin of 50%' + recordAndAssertResourceUsage("Query data leaf ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' - scenario | cpsPath || durationLimit | memoryLimit | expectedNumberOfValues - 'unique leaf value' | '/openroadm-devices/openroadm-device/@device-id' || 0.05 | 0.1 | OPENROADM_DEVICES_PER_ANCHOR - 'common leaf value' | '/openroadm-devices/openroadm-device/@ne-state' || 0.02 | 0.1 | 1 - 'non-existing data leaf' | '/openroadm-devices/openroadm-device/@non-existing' || 0.01 | 0.1 | 0 + scenario | cpsPath || expectedDuration | memoryLimit | expectedNumberOfValues + 'unique leaf value' | '/openroadm-devices/openroadm-device/@device-id' || 0.05 | 0.1 | OPENROADM_DEVICES_PER_ANCHOR + 'common leaf value' | '/openroadm-devices/openroadm-device/@ne-state' || 0.02 | 0.1 | 1 + 'non-existing data leaf' | '/openroadm-devices/openroadm-device/@non-existing' || 0.01 | 0.1 | 0 } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy index 183c634abc..0c66871790 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy @@ -75,7 +75,7 @@ class UpdatePerfTest extends CpsPerfTestBase { and: 'data leaves have expected values' assert totalNodes == countDataNodes(changeLeaves? '/openroadm-devices/openroadm-device[@status="fail"]' : '/openroadm-devices/openroadm-device[@status="success"]') - and: 'update completes within expected time and memory used is within limit' + and: 'update completes within expected time and a margin of 50%' recordAndAssertResourceUsage(scenario, expectedTime, resourceMeter.getTotalTimeInSeconds(), memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: scenario | totalNodes | startId | changeLeaves || expectedTime | memoryLimit @@ -99,7 +99,7 @@ class UpdatePerfTest extends CpsPerfTestBase { and: 'data leaves have expected values' assert totalNodes == countDataNodes(changeLeaves? '/openroadm-devices/openroadm-device[@status="fail"]' : '/openroadm-devices/openroadm-device[@status="success"]') - and: 'update completes within expected time and memory used is within limit' + and: 'update completes within expected time and a margin of 50%' recordAndAssertResourceUsage(scenario, expectedTime, resourceMeter.getTotalTimeInSeconds(), memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) @@ -107,7 +107,7 @@ class UpdatePerfTest extends CpsPerfTestBase { scenario | totalNodes | startId | changeLeaves || expectedTime | memoryLimit 'Replace list of 0 with 100' | 100 | 1 | false || 3.0 | 200 'Replace list of 100 using same data' | 100 | 1 | false || 2.0 | 200 - 'Replace list of 100 with new leaf values' | 100 | 1 | true || 2.3 | 200 + 'Replace list of 100 with new leaf values' | 100 | 1 | true || 2.0 | 200 'Replace list with 100 new nodes' | 100 | 101 | false || 6.1 | 200 'Replace list with 50 existing and 50 new' | 100 | 151 | true || 4.0 | 200 'Replace list of 100 nodes with 1' | 1 | 1 | false || 3.34 | 200 @@ -125,7 +125,7 @@ class UpdatePerfTest extends CpsPerfTestBase { resourceMeter.stop() then: 'data leaves have expected values' assert 100 == countDataNodes('/openroadm-devices/openroadm-device[@status="fail"]') - and: 'update completes within expected time and memory used is within limit' + and: 'update completes within expected time and a margin of 50%' recordAndAssertResourceUsage('Update leaves for 100 data nodes', 0.14, resourceMeter.getTotalTimeInSeconds(), 120, resourceMeter.getTotalMemoryUsageInMB()) } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy index 7c7a961f7c..21a9f19ea6 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy @@ -39,7 +39,7 @@ class WritePerfTest extends CpsPerfTestBase { resourceMeter.start() cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, jsonData, OffsetDateTime.now()) resourceMeter.stop() - then: 'the operation takes less than #expectedDuration and memory used is within limit' + then: 'the operation takes less than #expectedDuration with a margin of 50%' recordAndAssertResourceUsage("Writing ${totalNodes} devices", expectedDuration, resourceMeter.getTotalTimeInSeconds(), memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) @@ -64,7 +64,7 @@ class WritePerfTest extends CpsPerfTestBase { resourceMeter.start() cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, '/bookstore/categories[@code=1]', booksData, OffsetDateTime.now()) resourceMeter.stop() - then: 'the operation takes less than #expectedDuration and memory used is within limit' + then: 'the operation takes less than #expectedDuration with a margin of 50%' recordAndAssertResourceUsage("Writing ${totalBooks} books", expectedDuration, resourceMeter.totalTimeInSeconds, memoryLimit, resourceMeter.totalMemoryUsageInMB) @@ -92,7 +92,7 @@ class WritePerfTest extends CpsPerfTestBase { resourceMeter.start() cpsDataService.saveListElements(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, '/openroadm-devices', jsonListData, OffsetDateTime.now(), ContentType.JSON) resourceMeter.stop() - then: 'the operation takes less than #expectedDuration and memory used is within limit' + then: 'the operation takes less than #expectedDuration with a margin of 50%' recordAndAssertResourceUsage("Saving list of ${totalNodes} devices", expectedDuration, resourceMeter.totalTimeInSeconds, memoryLimit, resourceMeter.totalMemoryUsageInMB) diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy index 2f73023f07..c0dec8b53e 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy @@ -77,41 +77,38 @@ class CmHandleQueryPerfTest extends NcmpPerfTestBase { '/dmi-registry/cm-handles[@id="cm-' + it + '"]', OMIT_DESCENDANTS).size() } resourceMeter.stop() - then: + then: '100 data node have been found' assert count == 100 and: 'average performance is as expected' - def averageResponseTime = resourceMeter.totalTimeInSeconds / 100 - recordAndAssertResourceUsage('Look up CM-handle by id', 0.009, averageResponseTime, 15, resourceMeter.totalMemoryUsageInMB) + recordAndAssertResourceUsage('Look up CM-handle by id', 0.5, resourceMeter.totalTimeInSeconds, 15, resourceMeter.totalMemoryUsageInMB) } - def 'CM-handle is looked up by alternate-id.'() { - when: 'CM-handles are looked up by alternate-id 1000 times' + def 'CM-handle is looked up by alternate id.'() { + when: 'CM-handles are looked up by alternate id 100 times' int count = 0 resourceMeter.start() - (1..1000).each { + (1..100).each { count += cpsQueryService.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR, "/dmi-registry/cm-handles[@alternate-id='alt-${it}']", OMIT_DESCENDANTS).size() } resourceMeter.stop() - then: - assert count == 1000 + then: 'all alternate ids are resolved correctly' + assert count == 100 and: 'average performance is as expected' - def averageResponseTime = resourceMeter.totalTimeInSeconds / 1000 - recordAndAssertResourceUsage('Look up CM-handle by alternate-id', 0.02, averageResponseTime, 15, resourceMeter.totalMemoryUsageInMB) + recordAndAssertResourceUsage('Look up CM-handle by alternate-id', 1.75, resourceMeter.totalTimeInSeconds, 15, resourceMeter.totalMemoryUsageInMB) } - def 'A batch of CM-handles is looked up by alternate-id.'() { - given: 'a CPS Path Query to look up 100 alternate-ids in a single operation' + def 'A batch of CM-handles is looked up by alternate id.'() { + given: 'a CPS Path Query to look up 100 alternate ids in a single operation' def cpsPathQuery = '/dmi-registry/cm-handles[' + (1..100).collect { "@alternate-id='alt-${it}'" }.join(' or ') + ']' - when: 'CM-handles are looked up by alternate-ids in a single query' + when: 'CM-handles are looked up by alternate ids in a single query' resourceMeter.start() def count = cpsQueryService.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR, cpsPathQuery, OMIT_DESCENDANTS).size() resourceMeter.stop() then: 'expected amount of data was returned' assert count == 100 then: 'average performance is as expected' - def averageResponseTime = resourceMeter.totalTimeInSeconds / 100 - recordAndAssertResourceUsage('Batch look up CM-handle by alternate-id', 0.004, averageResponseTime, 15, resourceMeter.totalMemoryUsageInMB) + recordAndAssertResourceUsage('Batch look up CM-handle by alternate-id', 0.4, resourceMeter.totalTimeInSeconds, 15, resourceMeter.totalMemoryUsageInMB) } def 'Find any CM-handle given moduleSetTag when there are 20K READY handles with same moduleSetTag.'() {