- Removed average calculation where this lead to very small duration and therefore seemingly more variation
- Tweak a few expected duration to get closer to actual average
- Remove memory assertion from resource meter test since they are unreliable and cause the odd error
- Corrected wording on description related to limits
Issue-ID:CPS-2743
Change-Id: Id30bdbe7074297c32a236322fc66ee698fdd8e2b
Signed-off-by: ToineSiebelink <toine.siebelink@est.tech>
--- /dev/null
+/*
+ * ============LICENSE_START=======================================================
+ * Copyright (C) 2023-2025 OpenInfra Foundation Europe. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the 'License');
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an 'AS IS' BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.integration
+
+import spock.lang.Specification
+
+import java.util.concurrent.TimeUnit
+
+class ResourceMeterAccuracyTest extends Specification {
+
+ def objectUnderTest = new ResourceMeter()
+
+ def 'Resource meter duration measurement.'() {
+ when: 'we measure how long a known operation takes'
+ objectUnderTest.start()
+ TimeUnit.SECONDS.sleep(2)
+ objectUnderTest.stop()
+ then: 'the resource meter reports a duration within 20ms of the expected duration'
+ assert objectUnderTest.getTotalTimeInSeconds() >= 1.98
+ assert objectUnderTest.getTotalTimeInSeconds() <= 2.02
+ }
+
+}
+++ /dev/null
-/*
- * ============LICENSE_START=======================================================
- * Copyright (C) 2023 Nordix Foundation
- * ================================================================================
- * Licensed under the Apache License, Version 2.0 (the 'License');
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an 'AS IS' BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- * ============LICENSE_END=========================================================
- */
-
-package org.onap.cps.integration
-
-import java.util.concurrent.TimeUnit
-import spock.lang.Specification
-
-class ResourceMeterPerfTest extends Specification {
-
- final int MEGABYTE = 1_000_000
-
- def resourceMeter = new ResourceMeter()
-
- def 'ResourceMeter accurately measures duration'() {
- when: 'we measure how long a known operation takes'
- resourceMeter.start()
- TimeUnit.SECONDS.sleep(2)
- resourceMeter.stop()
- then: 'ResourceMeter reports a duration within 10ms of the expected duration'
- assert resourceMeter.getTotalTimeInSeconds() >= 2
- assert resourceMeter.getTotalTimeInSeconds() <= 2.01
- }
-
- def 'ResourceMeter reports memory usage when allocating a large byte array'() {
- when: 'the resource meter is started'
- resourceMeter.start()
- and: 'some memory is allocated'
- byte[] array = new byte[50 * MEGABYTE]
- and: 'the resource meter is stopped'
- resourceMeter.stop()
- then: 'the reported memory usage is close to the amount of memory allocated'
- assert resourceMeter.getTotalMemoryUsageInMB() >= 50
- assert resourceMeter.getTotalMemoryUsageInMB() <= 55
- }
-
- def 'ResourceMeter measures PEAK memory usage when garbage collector runs'() {
- when: 'the resource meter is started'
- resourceMeter.start()
- and: 'some memory is allocated'
- byte[] array = new byte[50 * MEGABYTE]
- and: 'the memory is garbage collected'
- array = null
- ResourceMeter.performGcAndWait()
- and: 'the resource meter is stopped'
- resourceMeter.stop()
- then: 'the reported memory usage is close to the peak amount of memory allocated'
- assert resourceMeter.getTotalMemoryUsageInMB() >= 50
- assert resourceMeter.getTotalMemoryUsageInMB() <= 55
- }
-
- def 'ResourceMeter measures memory increase only during measurement'() {
- given: '50 megabytes is allocated before measurement'
- byte[] arrayBefore = new byte[50 * MEGABYTE]
- when: 'memory is allocated during measurement'
- resourceMeter.start()
- byte[] arrayDuring = new byte[40 * MEGABYTE]
- resourceMeter.stop()
- and: '50 megabytes is allocated after measurement'
- byte[] arrayAfter = new byte[50 * MEGABYTE]
- then: 'the reported memory usage is close to the amount allocated DURING measurement'
- assert resourceMeter.getTotalMemoryUsageInMB() >= 40
- assert resourceMeter.getTotalMemoryUsageInMB() <= 45
- }
-
-}
static def LARGE_SCHEMA_SET = 'largeSchemaSet'
static def PERFORMANCE_RECORD = []
- static def DEFAULT_TIME_MARGIN = 1.5 // Allow 50% margin
- static def VERY_FAST_TEST_THRESHOLD = 0.01 // Defintion of a very vast test (hard to measure)
- static def DEFAULT_TIME_MARGIN_FOR_VERY_FAST_TEST = 3 // Allow 200% margin on very fast test (accuracy is an issue)
+ static def DEFAULT_TIME_LIMIT_FACTOR = 1.5 // Allow 50% margin on top of expected (average) value
+ static def VERY_FAST_TEST_THRESHOLD = 0.01 // Definition of a very vast test (hard to measure)
+ static def DEFAULT_TIME_LIMIT_FACTOR_FOR_VERY_FAST_TEST = 3 // Allow 200% margin on very fast test (accuracy is an issue)
def cleanupSpec() {
println('##################################################################################################')
abstract def createInitialData()
- def recordAndAssertResourceUsage(String shortTitle, double expectedTimeInSec, double recordedTimeInSec, memoryLimit, memoryUsageInMB, double timeMargin) {
- if (expectedTimeInSec <= VERY_FAST_TEST_THRESHOLD) {
- timeMargin = DEFAULT_TIME_MARGIN_FOR_VERY_FAST_TEST
+ def recordAndAssertResourceUsage(shortTitle, expectedAverageTimeInSec, recordedTimeInSec, memoryLimitCurrentlyNotAsserted, memoryUsageInMB, timeLimitFactor) {
+ if (expectedAverageTimeInSec <= VERY_FAST_TEST_THRESHOLD) {
+ timeLimitFactor = DEFAULT_TIME_LIMIT_FACTOR_FOR_VERY_FAST_TEST
}
- def testPassed = recordedTimeInSec <= timeMargin * expectedTimeInSec
+ def testPassed = recordedTimeInSec <= timeLimitFactor * expectedAverageTimeInSec
if (shortTitle.length() > 40) {
shortTitle = shortTitle.substring(0, 40)
}
- def record = String.format('%2d.%-40s limit %8.3f took %8.3f sec %,8.2f MB used ', PERFORMANCE_RECORD.size() + 1, shortTitle, expectedTimeInSec, recordedTimeInSec, memoryUsageInMB)
+ def record = String.format('%2d.%-40s limit %8.3f took %8.3f sec %,8.2f MB used ', PERFORMANCE_RECORD.size() + 1, shortTitle, expectedAverageTimeInSec, recordedTimeInSec, memoryUsageInMB)
record += testPassed ? 'PASS' : 'FAIL'
PERFORMANCE_RECORD.add(record)
- assert recordedTimeInSec <= timeMargin * expectedTimeInSec
+ assert recordedTimeInSec <= timeLimitFactor * expectedAverageTimeInSec
return true
}
def recordAndAssertResourceUsage(String shortTitle, double thresholdInSec, double recordedTimeInSec, memoryLimit, memoryUsageInMB) {
- recordAndAssertResourceUsage(shortTitle, thresholdInSec, recordedTimeInSec, memoryLimit, memoryUsageInMB, DEFAULT_TIME_MARGIN)
+ recordAndAssertResourceUsage(shortTitle, thresholdInSec, recordedTimeInSec, memoryLimit, memoryUsageInMB, DEFAULT_TIME_LIMIT_FACTOR)
}
}
/*
* ============LICENSE_START=======================================================
- * Copyright (C) 2022-2023 Nordix Foundation
+ * Copyright (C) 2022-2025 OpenInfra Foundation Europe. All rights reserved.
* ================================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
package org.onap.cps.integration.performance.cps
import org.onap.cps.api.CpsModuleService
-import org.onap.cps.integration.performance.base.CpsPerfTestBase
import org.onap.cps.api.model.ModuleReference
+import org.onap.cps.integration.performance.base.CpsPerfTestBase
import org.springframework.util.StopWatch
import java.util.concurrent.ThreadLocalRandom
stopWatch.stop()
assert newModuleReferences.size() > 0 && newModuleReferences.size() < 300
}
- assert stopWatch.getTotalTimeMillis() < 6000
+ assert stopWatch.getTotalTimeMillis() < 10_000
}
def createModuleReferencesWithRandomMatchingExistingModuleReferences() {
addAnchorsWithData(10, CPS_PERFORMANCE_TEST_DATASPACE, LARGE_SCHEMA_SET, 'delete', data, ContentType.JSON)
resourceMeter.stop()
def setupDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
- then: 'setup duration is within expected time and memory used is within limit'
+ then: 'setup duration is below accepted margin of the expected average'
recordAndAssertResourceUsage('Delete test setup', 103, setupDurationInSeconds, 800, resourceMeter.getTotalMemoryUsageInMB())
}
}
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
- then: 'delete duration is within expected time and memory used is within limit'
- recordAndAssertResourceUsage('Delete 100 containers', 1.5, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB())
+ then: 'delete duration is below accepted margin of the expected average'
+ recordAndAssertResourceUsage('Delete 100 containers', 1.4, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB())
}
def 'Batch delete 100 container nodes'() {
objectUnderTest.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'delete2', xpathsToDelete, OffsetDateTime.now())
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
- then: 'delete duration is within expected time and memory used is within limit'
+ then: 'delete duration is below accepted margin of the expected average'
recordAndAssertResourceUsage('Batch delete 100 containers', 0.65, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB())
}
}
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
- then: 'delete duration is within expected time and memory used is within limit'
+ then: 'delete duration is below accepted margin of the expected average'
recordAndAssertResourceUsage('Delete 100 lists elements', 1.5, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB())
}
objectUnderTest.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'delete4', xpathsToDelete, OffsetDateTime.now())
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
- then: 'delete duration is within expected time and memory used is within limit'
+ then: 'delete duration is below accepted margin of the expected average'
recordAndAssertResourceUsage('Batch delete 100 lists elements', 0.7, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB())
}
}
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
- then: 'delete duration is within expected time and memory used is within limit'
- recordAndAssertResourceUsage('Delete 100 whole lists', 3.2, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB(), 3)
+ then: 'delete duration is below accepted margin (4x) of the expected average'
+ recordAndAssertResourceUsage('Delete 100 whole lists', 3.2, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB(), 4)
}
def 'Batch delete 100 whole lists'() {
objectUnderTest.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'delete6', xpathsToDelete, OffsetDateTime.now())
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
- then: 'delete duration is within expected time and memory used is within limit'
+ then: 'delete duration is below accepted margin (3x) of the expected average'
recordAndAssertResourceUsage('Batch delete 100 whole lists', 3.4, deleteDurationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB(), 3)
}
objectUnderTest.deleteDataNode(CPS_PERFORMANCE_TEST_DATASPACE, 'delete7', '/openroadm-devices', OffsetDateTime.now())
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
- then: 'delete duration is within expected time and memory used is within limit'
+ then: 'delete duration is below accepted margin of the expected average'
recordAndAssertResourceUsage('Delete one large node', 2, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
}
objectUnderTest.deleteDataNode(CPS_PERFORMANCE_TEST_DATASPACE, 'delete8', '/', OffsetDateTime.now())
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
- then: 'delete duration is within expected time and memory used is within limit'
+ then: 'delete duration is below accepted margin of the expected average'
recordAndAssertResourceUsage('Delete root node', 2, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
}
objectUnderTest.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'delete9', OffsetDateTime.now())
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
- then: 'delete duration is within expected time and memory used is within limit'
+ then: 'delete duration is below accepted margin of the expected average'
recordAndAssertResourceUsage('Delete data nodes for anchor', 2, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
}
} catch (DataNodeNotFoundException ignored) {}
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
- then: 'delete duration is within expected time and memory used is within limit'
+ then: 'delete duration is below accepted margin of the expected average'
recordAndAssertResourceUsage('Batch delete 100 non-existing', 2, deleteDurationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB())
}
cpsAnchorService.deleteAnchors(CPS_PERFORMANCE_TEST_DATASPACE, anchorNames)
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
- then: 'delete duration is within expected time and memory used is within limit'
+ then: 'delete duration is below accepted margin of the expected average'
recordAndAssertResourceUsage('Delete test cleanup', 11.09, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
}
resourceMeter.stop()
assert countDataNodesInTree(result) == expectedNumberOfDataNodes
def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
- then: 'all data is read within #durationLimit ms and memory used is within limit'
- recordAndAssertResourceUsage("Read datatrees with ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+ then: 'all data is read within #expectedDuration seconds and a margin of 50%'
+ recordAndAssertResourceUsage("Read datatrees with ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
where: 'the following parameters are used'
- scenario | fetchDescendantsOption || durationLimit | memoryLimit | expectedNumberOfDataNodes
- 'no descendants' | OMIT_DESCENDANTS || 0.01 | 1 | 1
- 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.03 | 5 | 1 + OPENROADM_DEVICES_PER_ANCHOR
- 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.1 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
+ scenario | fetchDescendantsOption || expectedDuration | memoryLimit | expectedNumberOfDataNodes
+ 'no descendants' | OMIT_DESCENDANTS || 0.01 | 1 | 1
+ 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.03 | 5 | 1 + OPENROADM_DEVICES_PER_ANCHOR
+ 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.1 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
}
def 'Read data trees for multiple xpaths'() {
def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'requested nodes and their descendants are returned'
assert countDataNodesInTree(result) == OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
- and: 'all data is read within expected time and memory used is within limit'
- recordAndAssertResourceUsage("Read datatrees for multiple xpaths", 2.1, durationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB(),3)
+ and: 'all data is read within expected time and a margin of 200%'
+ recordAndAssertResourceUsage("Read datatrees for multiple xpaths", 1.1, durationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB(),3)
}
def 'Read for multiple xpaths to non-existing datanodes'() {
assert countDataNodesInTree(result) == expectedNumberOfDataNodes
resourceMeter.stop()
def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
- then: 'all data is read within expected time and memory used is within limit'
- recordAndAssertResourceUsage("Read datatrees using ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+ then: 'all data is read within expected time and a margin of 50%'
+ recordAndAssertResourceUsage("Read datatrees using ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
where: 'the following xpaths are used'
- scenario | xpath || durationLimit | memoryLimit | expectedNumberOfDataNodes
- 'openroadm root' | '/' || 1 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
- 'openroadm top element' | '/openroadm-devices' || 0.9 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
- 'openroadm whole list' | '/openroadm-devices/openroadm-device' || 1.06 | 250 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
+ scenario | xpath || expectedDuration | memoryLimit | expectedNumberOfDataNodes
+ 'openroadm root' | '/' || 1 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
+ 'openroadm top element' | '/openroadm-devices' || 0.9 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
+ 'openroadm whole list' | '/openroadm-devices/openroadm-device' || 1.06 | 250 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
}
}
def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'the expected number of nodes is returned'
assert countDataNodesInTree(result) == expectedNumberOfDataNodes
- and: 'all data is read within #durationLimit seconds and memory used is within limit'
- recordAndAssertResourceUsage("Query 1 anchor ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+ and: 'all data is read #expectedDuration seconds with a margin of 50%'
+ recordAndAssertResourceUsage("Query 1 anchor ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
where: 'the following parameters are used'
- scenario | cpsPath || durationLimit | memoryLimit | expectedNumberOfDataNodes
- 'top element' | '/openroadm-devices' || 0.89 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1
- 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || 1.3 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
- 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 1.1 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1
- 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 0.93 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1
- 'non-existing data' | '/path/to/non-existing/node[@id="1"]' || 0.01 | 1 | 0
+ scenario | cpsPath || expectedDuration | memoryLimit | expectedNumberOfDataNodes
+ 'top element' | '/openroadm-devices' || 0.89 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1
+ 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || 1.3 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
+ 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 1.1 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1
+ 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 0.93 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1
+ 'non-existing data' | '/path/to/non-existing/node[@id="1"]' || 0.01 | 1 | 0
}
def 'Query complete data trees across all anchors with #scenario.'() {
def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'the expected number of nodes is returned'
assert countDataNodesInTree(result) == expectedNumberOfDataNodes
- and: 'all data is read within #durationLimit seconds and memory used is within limit'
- recordAndAssertResourceUsage("Query across anchors ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+ and: 'all data is read #expectedDuration seconds with a margin of 50%'
+ recordAndAssertResourceUsage("Query across anchors ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
where: 'the following parameters are used'
- scenario | cpspath || durationLimit | memoryLimit | expectedNumberOfDataNodes
- 'top element' | '/openroadm-devices' || 3.0 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1)
- 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || 3.3 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE)
- 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 3.2 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1)
- 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 3.0 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1)
+ scenario | cpspath || expectedDuration | memoryLimit | expectedNumberOfDataNodes
+ 'top element' | '/openroadm-devices' || 3.0 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1)
+ 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || 3.3 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE)
+ 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 3.2 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1)
+ 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 3.0 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1)
}
def 'Query with leaf condition and #scenario.'() {
def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'the expected number of nodes is returned'
assert countDataNodesInTree(result) == expectedNumberOfDataNodes
- and: 'all data is read within #durationLimit seconds and memory used is within limit'
- recordAndAssertResourceUsage("Query with ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+ and: 'all data is read #expectedDuration seconds with a margin of 50%'
+ recordAndAssertResourceUsage("Query with ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
where: 'the following parameters are used'
- scenario | fetchDescendantsOption || durationLimit | memoryLimit | expectedNumberOfDataNodes
- 'no descendants' | OMIT_DESCENDANTS || 0.08 | 6 | OPENROADM_DEVICES_PER_ANCHOR
- 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.09 | 12 | OPENROADM_DEVICES_PER_ANCHOR * 2
- 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.05 | 200 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
+ scenario | fetchDescendantsOption || expectedDuration | memoryLimit | expectedNumberOfDataNodes
+ 'no descendants' | OMIT_DESCENDANTS || 0.08 | 6 | OPENROADM_DEVICES_PER_ANCHOR
+ 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.09 | 12 | OPENROADM_DEVICES_PER_ANCHOR * 2
+ 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.05 | 200 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
}
def 'Query ancestors with #scenario.'() {
def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'the expected number of nodes is returned'
assert countDataNodesInTree(result) == expectedNumberOfDataNodes
- and: 'all data is read within #durationLimit seconds and memory used is within limit'
- recordAndAssertResourceUsage("Query ancestors with ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+ and: 'all data is read #expectedDuration seconds with a margin of 50%'
+ recordAndAssertResourceUsage("Query ancestors with ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
where: 'the following parameters are used'
- scenario | fetchDescendantsOption || durationLimit | memoryLimit | expectedNumberOfDataNodes
- 'no descendants' | OMIT_DESCENDANTS || 0.06 | 3 | 1
- 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.07 | 8 | 1 + OPENROADM_DEVICES_PER_ANCHOR
- 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.0 | 400 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
+ scenario | fetchDescendantsOption || expectedDuration | memoryLimit | expectedNumberOfDataNodes
+ 'no descendants' | OMIT_DESCENDANTS || 0.06 | 3 | 1
+ 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.07 | 8 | 1 + OPENROADM_DEVICES_PER_ANCHOR
+ 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.0 | 400 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
}
def 'Query data leaf with #scenario.'() {
def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'the expected number of results is returned'
assert result.size() == expectedNumberOfValues
- and: 'all data is read within #durationLimit seconds and memory used is within limit'
- recordAndAssertResourceUsage("Query data leaf ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+ and: 'all data is read #expectedDuration seconds with a margin of 50%'
+ recordAndAssertResourceUsage("Query data leaf ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
where: 'the following parameters are used'
- scenario | cpsPath || durationLimit | memoryLimit | expectedNumberOfValues
- 'unique leaf value' | '/openroadm-devices/openroadm-device/@device-id' || 0.05 | 0.1 | OPENROADM_DEVICES_PER_ANCHOR
- 'common leaf value' | '/openroadm-devices/openroadm-device/@ne-state' || 0.02 | 0.1 | 1
- 'non-existing data leaf' | '/openroadm-devices/openroadm-device/@non-existing' || 0.01 | 0.1 | 0
+ scenario | cpsPath || expectedDuration | memoryLimit | expectedNumberOfValues
+ 'unique leaf value' | '/openroadm-devices/openroadm-device/@device-id' || 0.05 | 0.1 | OPENROADM_DEVICES_PER_ANCHOR
+ 'common leaf value' | '/openroadm-devices/openroadm-device/@ne-state' || 0.02 | 0.1 | 1
+ 'non-existing data leaf' | '/openroadm-devices/openroadm-device/@non-existing' || 0.01 | 0.1 | 0
}
}
and: 'data leaves have expected values'
assert totalNodes == countDataNodes(changeLeaves? '/openroadm-devices/openroadm-device[@status="fail"]'
: '/openroadm-devices/openroadm-device[@status="success"]')
- and: 'update completes within expected time and memory used is within limit'
+ and: 'update completes within expected time and a margin of 50%'
recordAndAssertResourceUsage(scenario, expectedTime, resourceMeter.getTotalTimeInSeconds(), memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
where:
scenario | totalNodes | startId | changeLeaves || expectedTime | memoryLimit
and: 'data leaves have expected values'
assert totalNodes == countDataNodes(changeLeaves? '/openroadm-devices/openroadm-device[@status="fail"]'
: '/openroadm-devices/openroadm-device[@status="success"]')
- and: 'update completes within expected time and memory used is within limit'
+ and: 'update completes within expected time and a margin of 50%'
recordAndAssertResourceUsage(scenario,
expectedTime, resourceMeter.getTotalTimeInSeconds(),
memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
scenario | totalNodes | startId | changeLeaves || expectedTime | memoryLimit
'Replace list of 0 with 100' | 100 | 1 | false || 3.0 | 200
'Replace list of 100 using same data' | 100 | 1 | false || 2.0 | 200
- 'Replace list of 100 with new leaf values' | 100 | 1 | true || 2.3 | 200
+ 'Replace list of 100 with new leaf values' | 100 | 1 | true || 2.0 | 200
'Replace list with 100 new nodes' | 100 | 101 | false || 6.1 | 200
'Replace list with 50 existing and 50 new' | 100 | 151 | true || 4.0 | 200
'Replace list of 100 nodes with 1' | 1 | 1 | false || 3.34 | 200
resourceMeter.stop()
then: 'data leaves have expected values'
assert 100 == countDataNodes('/openroadm-devices/openroadm-device[@status="fail"]')
- and: 'update completes within expected time and memory used is within limit'
+ and: 'update completes within expected time and a margin of 50%'
recordAndAssertResourceUsage('Update leaves for 100 data nodes', 0.14, resourceMeter.getTotalTimeInSeconds(), 120, resourceMeter.getTotalMemoryUsageInMB())
}
resourceMeter.start()
cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, jsonData, OffsetDateTime.now())
resourceMeter.stop()
- then: 'the operation takes less than #expectedDuration and memory used is within limit'
+ then: 'the operation takes less than #expectedDuration with a margin of 50%'
recordAndAssertResourceUsage("Writing ${totalNodes} devices",
expectedDuration, resourceMeter.getTotalTimeInSeconds(),
memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
resourceMeter.start()
cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, '/bookstore/categories[@code=1]', booksData, OffsetDateTime.now())
resourceMeter.stop()
- then: 'the operation takes less than #expectedDuration and memory used is within limit'
+ then: 'the operation takes less than #expectedDuration with a margin of 50%'
recordAndAssertResourceUsage("Writing ${totalBooks} books",
expectedDuration, resourceMeter.totalTimeInSeconds,
memoryLimit, resourceMeter.totalMemoryUsageInMB)
resourceMeter.start()
cpsDataService.saveListElements(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, '/openroadm-devices', jsonListData, OffsetDateTime.now(), ContentType.JSON)
resourceMeter.stop()
- then: 'the operation takes less than #expectedDuration and memory used is within limit'
+ then: 'the operation takes less than #expectedDuration with a margin of 50%'
recordAndAssertResourceUsage("Saving list of ${totalNodes} devices",
expectedDuration, resourceMeter.totalTimeInSeconds,
memoryLimit, resourceMeter.totalMemoryUsageInMB)
'/dmi-registry/cm-handles[@id="cm-' + it + '"]', OMIT_DESCENDANTS).size()
}
resourceMeter.stop()
- then:
+ then: '100 data node have been found'
assert count == 100
and: 'average performance is as expected'
- def averageResponseTime = resourceMeter.totalTimeInSeconds / 100
- recordAndAssertResourceUsage('Look up CM-handle by id', 0.009, averageResponseTime, 15, resourceMeter.totalMemoryUsageInMB)
+ recordAndAssertResourceUsage('Look up CM-handle by id', 0.5, resourceMeter.totalTimeInSeconds, 15, resourceMeter.totalMemoryUsageInMB)
}
- def 'CM-handle is looked up by alternate-id.'() {
- when: 'CM-handles are looked up by alternate-id 1000 times'
+ def 'CM-handle is looked up by alternate id.'() {
+ when: 'CM-handles are looked up by alternate id 100 times'
int count = 0
resourceMeter.start()
- (1..1000).each {
+ (1..100).each {
count += cpsQueryService.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR,
"/dmi-registry/cm-handles[@alternate-id='alt-${it}']", OMIT_DESCENDANTS).size()
}
resourceMeter.stop()
- then:
- assert count == 1000
+ then: 'all alternate ids are resolved correctly'
+ assert count == 100
and: 'average performance is as expected'
- def averageResponseTime = resourceMeter.totalTimeInSeconds / 1000
- recordAndAssertResourceUsage('Look up CM-handle by alternate-id', 0.02, averageResponseTime, 15, resourceMeter.totalMemoryUsageInMB)
+ recordAndAssertResourceUsage('Look up CM-handle by alternate-id', 1.75, resourceMeter.totalTimeInSeconds, 15, resourceMeter.totalMemoryUsageInMB)
}
- def 'A batch of CM-handles is looked up by alternate-id.'() {
- given: 'a CPS Path Query to look up 100 alternate-ids in a single operation'
+ def 'A batch of CM-handles is looked up by alternate id.'() {
+ given: 'a CPS Path Query to look up 100 alternate ids in a single operation'
def cpsPathQuery = '/dmi-registry/cm-handles[' + (1..100).collect { "@alternate-id='alt-${it}'" }.join(' or ') + ']'
- when: 'CM-handles are looked up by alternate-ids in a single query'
+ when: 'CM-handles are looked up by alternate ids in a single query'
resourceMeter.start()
def count = cpsQueryService.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR, cpsPathQuery, OMIT_DESCENDANTS).size()
resourceMeter.stop()
then: 'expected amount of data was returned'
assert count == 100
then: 'average performance is as expected'
- def averageResponseTime = resourceMeter.totalTimeInSeconds / 100
- recordAndAssertResourceUsage('Batch look up CM-handle by alternate-id', 0.004, averageResponseTime, 15, resourceMeter.totalMemoryUsageInMB)
+ recordAndAssertResourceUsage('Batch look up CM-handle by alternate-id', 0.4, resourceMeter.totalTimeInSeconds, 15, resourceMeter.totalMemoryUsageInMB)
}
def 'Find any CM-handle given moduleSetTag when there are 20K READY handles with same moduleSetTag.'() {
Code Review / cps.git / commitdiff