addAnchorsWithData(OPENROADM_ANCHORS, CPS_PERFORMANCE_TEST_DATASPACE, LARGE_SCHEMA_SET, 'openroadm', data, ContentType.JSON)
resourceMeter.stop()
def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
- recordAndAssertResourceUsage('CPS:Creating openroadm anchors with large data tree', 'Creating openroadm anchors with large data tree', 100, durationInSeconds, 600, resourceMeter.getTotalMemoryUsageInMB(), false)
+ recordAndAssertResourceUsage('CPS:Creating openroadm anchors with large data tree', 100, durationInSeconds, resourceMeter.getTotalMemoryUsageInMB(), false)
}
def generateOpenRoadData(numberOfNodes) {
resourceMeter.stop()
then: 'expected data exists'
assert result.xpath == ['/openroadm-devices']
- then: 'operation completes within expected time'
- recordAndAssertResourceUsage('CPS:Preload test data', 'CPS pre-load test data', 100, resourceMeter.totalTimeInSeconds, 600, resourceMeter.totalMemoryUsageInMB)
}
}
resourceMeter.stop()
then: 'expected data exists'
assert result.xpath == [REGISTRY_PARENT]
- and: 'operation completes within expected time'
- recordAndAssertResourceUsage('NCMP:Pre-load test data', 'NCMP pre-load test data', 15, resourceMeter.totalTimeInSeconds, 600, resourceMeter.totalMemoryUsageInMB)
}
}
abstract class PerfTestBase extends CpsIntegrationSpecBase {
static def LARGE_SCHEMA_SET = 'largeSchemaSet'
- static def PERFORMANCE_RECORDS = []
+ static def PERFORMANCE_TXT_RECORDS = []
static def PERFORMANCE_CSV_RECORDS = []
static def DEFAULT_TIME_LIMIT_FACTOR = 2 // Allow 100% margin on top of expected (average) value
static def VERY_FAST_TEST_THRESHOLD = 0.01 // Definition of a very vast test (hard to measure)
static def CSV_PREFIX = "### CSV ### " // To be used in plot-job to extract CSV data from logs
static def TIME_FORMAT = new DecimalFormat("0.####")
-
def cleanupSpec() {
printTextRecords()
printCsvRecordsWithPrefix()
abstract def createInitialData()
- def recordAndAssertResourceUsage(String title, String shortTitle, double expectedAverageTimeInSec, double recordedTimeInSec, memoryLimitCurrentlyNotAsserted, double memoryUsageInMB, boolean referenceGraph) {
+ def recordAndAssertResourceUsage(String title, double expectedAverageTimeInSec, double recordedTimeInSec, double memoryUsageInMB, boolean referenceGraph) {
def timeLimitFactor = DEFAULT_TIME_LIMIT_FACTOR
if (expectedAverageTimeInSec <= VERY_FAST_TEST_THRESHOLD) {
timeLimitFactor = DEFAULT_TIME_LIMIT_FACTOR_FOR_VERY_FAST_TEST
}
def testPassed = recordedTimeInSec <= timeLimitFactor * expectedAverageTimeInSec
- addRecord(shortTitle, expectedAverageTimeInSec, recordedTimeInSec, memoryUsageInMB, testPassed)
+ addRecord(title, expectedAverageTimeInSec, recordedTimeInSec, memoryUsageInMB, testPassed)
if (referenceGraph) {
addCsvRecord('Reference', title, expectedAverageTimeInSec, recordedTimeInSec)
}
return true
}
- def recordAndAssertResourceUsage(String title, String shortTitle, double expectedAverageTimeInSec, double recordedTimeInSec, memoryLimitCurrentlyNotAsserted, double memoryUsageInMB) {
- recordAndAssertResourceUsage(title, shortTitle, expectedAverageTimeInSec, recordedTimeInSec, memoryLimitCurrentlyNotAsserted, memoryUsageInMB, false)
+ def recordAndAssertResourceUsage(String title, double expectedAverageTimeInSec, double recordedTimeInSec, double memoryUsageInMB) {
+ recordAndAssertResourceUsage(title, expectedAverageTimeInSec, recordedTimeInSec, memoryUsageInMB, false)
}
def addRecord(shortTitle, double expectedAverageTimeInSec, double recordedTimeInSec, double memoryUsageInMB, boolean testPassed) {
if (shortTitle.length() > 40) {
shortTitle = shortTitle.substring(0, 40)
}
- def record = String.format('%2d.%-40s limit %8.3f took %8.3f sec %,8.2f MB used ', PERFORMANCE_RECORDS.size() + 1, shortTitle, expectedAverageTimeInSec, recordedTimeInSec, memoryUsageInMB)
+ def record = String.format('%2d.%-40s limit %8.3f took %8.3f sec %,8.2f MB used ', PERFORMANCE_TXT_RECORDS.size() + 1, shortTitle, expectedAverageTimeInSec, recordedTimeInSec, memoryUsageInMB)
record += testPassed ? 'PASS' : 'FAIL'
- PERFORMANCE_RECORDS.add(record)
+ PERFORMANCE_TXT_RECORDS.add(record)
}
-
-
def addCsvRecord(tabName, title, double expectedAverageTimeInSec, double recordedTimeInSec) {
def csvRecord = String.format('%s,%s,%s,%s', tabName, title, TIME_FORMAT.format(expectedAverageTimeInSec), TIME_FORMAT.format(recordedTimeInSec))
PERFORMANCE_CSV_RECORDS.add(csvRecord)
println('##################################################################################################')
printTitle()
println('##################################################################################################')
- PERFORMANCE_RECORDS.sort().each {
+ PERFORMANCE_TXT_RECORDS.sort().each {
println(it)
}
- PERFORMANCE_RECORDS.clear()
+ PERFORMANCE_TXT_RECORDS.clear()
}
def printCsvRecordsWithPrefix() {
resourceMeter.stop()
def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'the operation completes within 16 seconds'
- recordAndAssertResourceUsage('CPS:Creating 33,000 books', "Creating 33,000 books", 16, durationInSeconds, 150, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH)
+ recordAndAssertResourceUsage('CPS:Creating 33,000 books', 16, durationInSeconds, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH)
}
def 'Get data nodes from multiple xpaths 32K (2^15) limit exceeded.'() {
resourceMeter.stop()
def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'test data is deleted in 1 second'
- recordAndAssertResourceUsage('CPS:Deleting test data', "Deleting test data", 0.068, durationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Deleting test data', 0.068, durationInSeconds, resourceMeter.getTotalMemoryUsageInMB())
}
def countDataNodes() {
stopWatch.stop()
assert newModuleReferences.size() > 0 && newModuleReferences.size() < 300
}
- recordAndAssertResourceUsage('CPS:Create large schema set', 'Create large schema set', 2, stopWatch.getTotalTimeSeconds(), 3, 0, REFERENCE_GRAPH)
+ recordAndAssertResourceUsage('CPS:Create large schema set', 1, stopWatch.getTotalTimeSeconds(), 0, REFERENCE_GRAPH)
}
def createModuleReferencesWithRandomMatchingExistingModuleReferences() {
resourceMeter.stop()
def setupDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'setup duration is below accepted margin of the expected average'
- recordAndAssertResourceUsage('CPS:Delete test setup', 'Delete test setup', 100, setupDurationInSeconds, 800, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Delete test setup', 85, setupDurationInSeconds, resourceMeter.getTotalMemoryUsageInMB())
}
def 'Delete 100 container nodes'() {
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'delete duration is below accepted margin of the expected average'
- recordAndAssertResourceUsage('CPS:Delete 100 containers', 'Delete 100 containers', 2.1, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Delete 100 containers', 2.1, deleteDurationInSeconds, resourceMeter.getTotalMemoryUsageInMB())
}
def 'Batch delete 100 container nodes'() {
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'delete duration is below accepted margin of the expected average'
- recordAndAssertResourceUsage('CPS:Batch delete 100 containers', 'Batch delete 100 containers', 0.65, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Batch delete 100 containers', 0.65, deleteDurationInSeconds, resourceMeter.getTotalMemoryUsageInMB())
}
def 'Delete 100 list elements'() {
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'delete duration is below accepted margin of the expected average'
- recordAndAssertResourceUsage('CPS:Delete 100 lists elements', 'Delete 100 lists elements', 2.3, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH)
+ recordAndAssertResourceUsage('CPS:Delete 100 lists elements', 2.3, deleteDurationInSeconds, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH)
}
def 'Batch delete 100 list elements'() {
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'delete duration is below accepted margin of the expected average'
- recordAndAssertResourceUsage('CPS:Batch delete 100 lists elements', 'Batch delete 100 lists elements', 0.7, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Batch delete 100 lists elements', 0.7, deleteDurationInSeconds, resourceMeter.getTotalMemoryUsageInMB())
}
def 'Delete 100 whole lists'() {
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'delete duration is below accepted margin (4x) of the expected average'
- recordAndAssertResourceUsage('CPS:Delete 100 whole lists', 'Delete 100 whole lists', 3.2, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH)
+ recordAndAssertResourceUsage('CPS:Delete 100 whole lists', 3.2, deleteDurationInSeconds, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH)
}
def 'Batch delete 100 whole lists'() {
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'delete duration is below accepted margin (3x) of the expected average'
- recordAndAssertResourceUsage('CPS:Batch delete 100 whole lists', 'Batch delete 100 whole lists', 2.4, deleteDurationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH)
+ recordAndAssertResourceUsage('CPS:Batch delete 100 whole lists', 2.4, deleteDurationInSeconds, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH)
}
def 'Delete 1 large data node'() {
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'delete duration is below accepted margin of the expected average'
- recordAndAssertResourceUsage('CPS:Delete one large node', 'Delete one large node', 1.9, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Delete one large node', 1.9, deleteDurationInSeconds, resourceMeter.getTotalMemoryUsageInMB())
}
def 'Delete root node with many descendants'() {
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'delete duration is below accepted margin of the expected average'
- recordAndAssertResourceUsage('CPS:Delete root node', 'Delete root node', 1.9, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Delete root node', 1.9, deleteDurationInSeconds, resourceMeter.getTotalMemoryUsageInMB())
}
def 'Delete data nodes for an anchor'() {
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'delete duration is below accepted margin of the expected average'
- recordAndAssertResourceUsage('CPS:Delete data nodes for anchor', 'Delete data nodes for anchor', 1.9, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Delete data nodes for anchor', 1.9, deleteDurationInSeconds, resourceMeter.getTotalMemoryUsageInMB())
}
def 'Batch delete 100 non-existing nodes'() {
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'delete duration is below accepted margin of the expected average'
- recordAndAssertResourceUsage('CPS:Batch delete 100 non-existing data nodes', 'Batch delete 100 non-existing', 2, deleteDurationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Batch delete 100 non-existing data nodes', 2, deleteDurationInSeconds, resourceMeter.getTotalMemoryUsageInMB())
}
def 'Clean up test data'() {
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'delete duration is below accepted margin of the expected average'
- recordAndAssertResourceUsage('CPS:Delete test cleanup', 'Delete test cleanup', 10, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Delete test cleanup', 9, deleteDurationInSeconds, resourceMeter.getTotalMemoryUsageInMB())
}
}
assert countDataNodesInTree(result) == expectedNumberOfDataNodes
def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'all data is read within #expectedDuration seconds and a margin of 100%'
- recordAndAssertResourceUsage("CPS:Read datatree with ${scenario}", "Read datatrees with ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB(), referencegraph)
+ recordAndAssertResourceUsage("CPS:Read datatree with ${scenario}", expectedDuration, durationInSeconds, resourceMeter.getTotalMemoryUsageInMB(), referencegraph)
where: 'the following parameters are used'
- scenario | fetchDescendantsOption | referencegraph || expectedDuration | memoryLimit | expectedNumberOfDataNodes
- 'no descendants' | OMIT_DESCENDANTS | false || 0.01 | 1 | 1
- 'direct descendants' | DIRECT_CHILDREN_ONLY | false || 0.045 | 5 | 1 + OPENROADM_DEVICES_PER_ANCHOR
- 'all descendants' | INCLUDE_ALL_DESCENDANTS | true || 1.1 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
+ scenario | fetchDescendantsOption | referencegraph || expectedDuration | expectedNumberOfDataNodes
+ 'no descendants' | OMIT_DESCENDANTS | false || 0.01 | 1
+ 'direct descendants' | DIRECT_CHILDREN_ONLY | false || 0.045 | 1 + OPENROADM_DEVICES_PER_ANCHOR
+ 'all descendants' | INCLUDE_ALL_DESCENDANTS | true || 1.1 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
}
def 'Read data trees for multiple xpaths'() {
then: 'requested nodes and their descendants are returned'
assert countDataNodesInTree(result) == OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
and: 'all data is read within expected time and a margin of 200%'
- recordAndAssertResourceUsage('CPS:Read datatrees for multiple xpaths', "Read datatrees for multiple xpaths", 2.0, durationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Read datatrees for multiple xpaths', 2.0, durationInSeconds, resourceMeter.getTotalMemoryUsageInMB())
}
def 'Read for multiple xpaths to non-existing datanodes'() {
then: 'no data is returned'
assert result.isEmpty()
and: 'the operation completes within within expected time'
- recordAndAssertResourceUsage('CPS:Read non-existing xpaths', "Read non-existing xpaths", 0.06, durationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Read non-existing xpaths', 0.06, durationInSeconds, resourceMeter.getTotalMemoryUsageInMB())
}
def 'Read complete data trees using #scenario.'() {
resourceMeter.stop()
def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'all data is read within expected time and a margin of 100%'
- recordAndAssertResourceUsage("CPS:Read datatrees using ${scenario}", "Read datatrees using ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage("CPS:Read datatrees using ${scenario}", expectedDuration, durationInSeconds, resourceMeter.getTotalMemoryUsageInMB())
where: 'the following xpaths are used'
- scenario | xpath || expectedDuration | memoryLimit | expectedNumberOfDataNodes
- 'openroadm root' | '/' || 1 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
- 'openroadm top element' | '/openroadm-devices' || 0.95 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
- 'openroadm whole list' | '/openroadm-devices/openroadm-device' || 1.06 | 250 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
+ scenario | xpath || expectedDuration | expectedNumberOfDataNodes
+ 'openroadm root' | '/' || 1 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
+ 'openroadm top element' | '/openroadm-devices' || 0.95 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
+ 'openroadm whole list' | '/openroadm-devices/openroadm-device' || 1.06 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
}
}
}
resourceMeter.stop()
then: 'operation takes less than expected duration'
- recordAndAssertResourceUsage('CPS:Module query test setup', 'Module query test setup', 150, resourceMeter.totalTimeInSeconds, 500, resourceMeter.totalMemoryUsageInMB)
+ recordAndAssertResourceUsage('CPS:Module query test setup', 150, resourceMeter.totalTimeInSeconds, resourceMeter.totalMemoryUsageInMB)
}
def 'Querying anchors by module name is NOT dependant on the file size of the module.'() {
then: 'expected number of anchors is returned'
assert result.size() == TOTAL_TEST_ANCHORS
and: 'operation completes with expected resource usage'
- recordAndAssertResourceUsage("CPS:Query for anchors with ${scenario}", "Query for anchors with ${scenario}", 0.1, resourceMeter.totalTimeInSeconds, 5, resourceMeter.totalMemoryUsageInMB, REFERENCE_GRAPH)
+ recordAndAssertResourceUsage("CPS:Query for anchors with ${scenario}", 0.1, resourceMeter.totalTimeInSeconds, resourceMeter.totalMemoryUsageInMB, REFERENCE_GRAPH)
where: 'the following parameters are used'
scenario | yangModuleName
'1 KB module' | 'module0'
then: 'the expected number of nodes is returned'
assert countDataNodesInTree(result) == expectedNumberOfDataNodes
and: 'all data is read #expectedDuration seconds with a margin of 100%'
- recordAndAssertResourceUsage("CPS:Query 1 anchor ${scenario}", "Query 1 anchor ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage("CPS:Query 1 anchor ${scenario}", expectedDuration, durationInSeconds, resourceMeter.getTotalMemoryUsageInMB())
where: 'the following parameters are used'
- scenario | cpsPath || expectedDuration | memoryLimit | expectedNumberOfDataNodes
- 'top element' | '/openroadm-devices' || 0.89 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1
- 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || 1.3 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
- 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 1.3 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1
- 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 0.93 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1
- 'non-existing data' | '/path/to/non-existing/node[@id="1"]' || 0.01 | 1 | 0
+ scenario | cpsPath || expectedDuration | expectedNumberOfDataNodes
+ 'top element' | '/openroadm-devices' || 0.89 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1
+ 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || 1.1 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
+ 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 1.3 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1
+ 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 0.93 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1
+ 'non-existing data' | '/path/to/non-existing/node[@id="1"]' || 0.01 | 0
}
def 'Query complete data trees across all anchors with #scenario.'() {
then: 'the expected number of nodes is returned'
assert countDataNodesInTree(result) == expectedNumberOfDataNodes
and: 'all data is read #expectedDuration seconds with a margin of 100%'
- recordAndAssertResourceUsage("CPS:Query across anchors ${scenario}", "Query across anchors ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH)
+ recordAndAssertResourceUsage("CPS:Query across anchors ${scenario}", expectedDuration, durationInSeconds, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH)
where: 'the following parameters are used'
- scenario | cpspath || expectedDuration | memoryLimit | expectedNumberOfDataNodes
- 'top element' | '/openroadm-devices' || 3.0 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1)
- 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || 3.3 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE)
- 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 3.2 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1)
- 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 3.0 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1)
+ scenario | cpspath || expectedDuration | expectedNumberOfDataNodes
+ 'top element' | '/openroadm-devices' || 3.0 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1)
+ 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || 3.1 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE)
+ 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 3.2 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1)
+ 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 3.0 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1)
}
def 'Query with leaf condition and #scenario.'() {
then: 'the expected number of nodes is returned'
assert countDataNodesInTree(result) == expectedNumberOfDataNodes
and: 'all data is read #expectedDuration seconds with a margin of 100%'
- recordAndAssertResourceUsage("CPS:Query with ${scenario}", "Query with ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage("CPS:Query with ${scenario}", expectedDuration, durationInSeconds, resourceMeter.getTotalMemoryUsageInMB())
where: 'the following parameters are used'
- scenario | fetchDescendantsOption || expectedDuration | memoryLimit | expectedNumberOfDataNodes
- 'no descendants' | OMIT_DESCENDANTS || 0.09 | 6 | OPENROADM_DEVICES_PER_ANCHOR
- 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.12 | 12 | OPENROADM_DEVICES_PER_ANCHOR * 2
- 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.05 | 200 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
+ scenario | fetchDescendantsOption || expectedDuration | expectedNumberOfDataNodes
+ 'no descendants' | OMIT_DESCENDANTS || 0.09 | OPENROADM_DEVICES_PER_ANCHOR
+ 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.12 | OPENROADM_DEVICES_PER_ANCHOR * 2
+ 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.05 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
}
def 'Query ancestors with #scenario.'() {
then: 'the expected number of nodes is returned'
assert countDataNodesInTree(result) == expectedNumberOfDataNodes
and: 'all data is read #expectedDuration seconds with a margin of 100%'
- recordAndAssertResourceUsage("CPS:Query ancestors with ${scenario}", "Query ancestors with ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage("CPS:Query ancestors with ${scenario}", expectedDuration, durationInSeconds, resourceMeter.getTotalMemoryUsageInMB())
where: 'the following parameters are used'
- scenario | fetchDescendantsOption || expectedDuration | memoryLimit | expectedNumberOfDataNodes
- 'no descendants' | OMIT_DESCENDANTS || 0.08 | 3 | 1
- 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.09 | 8 | 1 + OPENROADM_DEVICES_PER_ANCHOR
- 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.0 | 400 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
+ scenario | fetchDescendantsOption || expectedDuration | expectedNumberOfDataNodes
+ 'no descendants' | OMIT_DESCENDANTS || 0.08 | 1
+ 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.09 | 1 + OPENROADM_DEVICES_PER_ANCHOR
+ 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.0 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
}
def 'Query data leaf with #scenario.'() {
then: 'the expected number of results is returned'
assert result.size() == expectedNumberOfValues
and: 'all data is read #expectedDuration seconds with a margin of 100%'
- recordAndAssertResourceUsage("CPS:Query data leaf ${scenario}", "Query data leaf ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage("CPS:Query data leaf ${scenario}", expectedDuration, durationInSeconds, resourceMeter.getTotalMemoryUsageInMB())
where: 'the following parameters are used'
- scenario | cpsPath || expectedDuration | memoryLimit | expectedNumberOfValues
- 'unique leaf value' | '/openroadm-devices/openroadm-device/@device-id' || 0.05 | 0.1 | OPENROADM_DEVICES_PER_ANCHOR
- 'common leaf value' | '/openroadm-devices/openroadm-device/@ne-state' || 0.02 | 0.1 | 1
- 'non-existing data leaf' | '/openroadm-devices/openroadm-device/@non-existing' || 0.01 | 0.1 | 0
+ scenario | cpsPath || expectedDuration | expectedNumberOfValues
+ 'unique leaf value' | '/openroadm-devices/openroadm-device/@device-id' || 0.05 | OPENROADM_DEVICES_PER_ANCHOR
+ 'common leaf value' | '/openroadm-devices/openroadm-device/@ne-state' || 0.013 | 1
+ 'non-existing data leaf' | '/openroadm-devices/openroadm-device/@non-existing' || 0.01 | 0
}
}
assert totalNodes == countDataNodes(changeLeaves? '/openroadm-devices/openroadm-device[@status="fail"]'
: '/openroadm-devices/openroadm-device[@status="success"]')
and: 'update completes within expected time and a margin of 100%'
- recordAndAssertResourceUsage("CPS:${scenario}", scenario, expectedTime, resourceMeter.getTotalTimeInSeconds(), memoryLimit, resourceMeter.getTotalMemoryUsageInMB(), referenceGraph)
+ recordAndAssertResourceUsage("CPS:${scenario}", expectedTime, resourceMeter.getTotalTimeInSeconds(), resourceMeter.getTotalMemoryUsageInMB(), referenceGraph)
where:
- scenario | totalNodes | startId | changeLeaves || expectedTime | memoryLimit | referenceGraph
- 'Replace 0 nodes with 100' | 100 | 1 | false || 3.0 | 200 | false
- 'Replace 100 using same data' | 100 | 1 | false || 5.5 | 200 | false
- 'Replace 100 with new leaf values' | 100 | 1 | true || 5.4 | 200 | false
- 'Replace 100 with 100 new nodes' | 100 | 101 | false || 9.3 | 200 | true
- 'Replace 50 existing and 50 new' | 100 | 151 | true || 7.3 | 200 | false
- 'Replace 100 nodes with 0' | 0 | 1 | false || 6.0 | 200 | false
+ scenario | totalNodes | startId | changeLeaves || expectedTime | referenceGraph
+ 'Replace 0 nodes with 100' | 100 | 1 | false || 3.0 | false
+ 'Replace 100 using same data' | 100 | 1 | false || 5.0 | false
+ 'Replace 100 with new leaf values' | 100 | 1 | true || 5.4 | false
+ 'Replace 100 with 100 new nodes' | 100 | 101 | false || 9.3 | true
+ 'Replace 50 existing and 50 new' | 100 | 151 | true || 7.3 | false
+ 'Replace 100 nodes with 0' | 0 | 1 | false || 6.0 | false
}
def 'Replace list content: #scenario.'() {
assert totalNodes == countDataNodes(changeLeaves? '/openroadm-devices/openroadm-device[@status="fail"]'
: '/openroadm-devices/openroadm-device[@status="success"]')
and: 'update completes within expected time and a margin of 100%'
- recordAndAssertResourceUsage("CPS:${scenario}", scenario, expectedTime, resourceMeter.getTotalTimeInSeconds(), memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage("CPS:${scenario}", expectedTime, resourceMeter.getTotalTimeInSeconds(), resourceMeter.getTotalMemoryUsageInMB())
where:
- scenario | totalNodes | startId | changeLeaves || expectedTime | memoryLimit
- 'Replace list of 0 with 100' | 100 | 1 | false || 3.0 | 200
- 'Replace list of 100 using same data' | 100 | 1 | false || 5.5 | 200
- 'Replace list of 100 with new leaf values' | 100 | 1 | true || 5.5 | 200
- 'Replace list with 100 new nodes' | 100 | 101 | false || 9.0 | 200
- 'Replace list with 50 existing and 50 new' | 100 | 151 | true || 8.0 | 200
- 'Replace list of 100 nodes with 1' | 1 | 1 | false || 5.9 | 200
+ scenario | totalNodes | startId | changeLeaves || expectedTime
+ 'Replace list of 0 with 100' | 100 | 1 | false || 3.0
+ 'Replace list of 100 using same data' | 100 | 1 | false || 5.0
+ 'Replace list of 100 with new leaf values' | 100 | 1 | true || 5.4
+ 'Replace list with 100 new nodes' | 100 | 101 | false || 9.0
+ 'Replace list with 50 existing and 50 new' | 100 | 151 | true || 7.0
+ 'Replace list of 100 nodes with 1' | 1 | 1 | false || 5.9
}
def 'Update leaves for 100 data nodes.'() {
then: 'data leaves have expected values'
assert 100 == countDataNodes('/openroadm-devices/openroadm-device[@status="fail"]')
and: 'update completes within expected time and a margin of 100%'
- recordAndAssertResourceUsage('CPS:Update leaves for 100 data nodes', 'Update leaves for 100 data nodes', 0.22, resourceMeter.getTotalTimeInSeconds(), 120, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Update leaves for 100 data nodes', 0.22, resourceMeter.getTotalTimeInSeconds(), resourceMeter.getTotalMemoryUsageInMB())
}
def 'Clean up for CPS Update API.'() {
cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, jsonData, OffsetDateTime.now())
resourceMeter.stop()
then: 'the operation takes less than #expectedDuration with a margin of 100%'
- recordAndAssertResourceUsage("CPS:Writing ${totalNodes} devices", "Writing ${totalNodes} devices", expectedDuration, resourceMeter.getTotalTimeInSeconds(), memoryLimit, resourceMeter.getTotalMemoryUsageInMB(), referenceGraph)
+ recordAndAssertResourceUsage("CPS:Writing ${totalNodes} devices", expectedDuration, resourceMeter.getTotalTimeInSeconds(), resourceMeter.getTotalMemoryUsageInMB(), referenceGraph)
cleanup:
cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR)
where:
- totalNodes || expectedDuration | memoryLimit | referenceGraph
- 50 || 1.45 | 100 | false
- 100 || 2.9 | 200 | false
- 200 || 6.2 | 400 | true
- 400 || 13.0 | 500 | false
+ totalNodes || expectedDuration | referenceGraph
+ 50 || 1.45 | false
+ 100 || 2.9 | false
+ 200 || 6.2 | true
+ 400 || 13.0 | false
}
def 'Writing bookstore data has exponential time.'() {
cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, '/bookstore/categories[@code=1]', booksData, OffsetDateTime.now())
resourceMeter.stop()
then: 'the operation takes less than #expectedDuration with a margin of 100%'
- recordAndAssertResourceUsage("CPS:Writing ${totalBooks} books", "Writing ${totalBooks} books", expectedDuration, resourceMeter.totalTimeInSeconds, memoryLimit, resourceMeter.totalMemoryUsageInMB, referenceGraph)
+ recordAndAssertResourceUsage("CPS:Writing ${totalBooks} books", expectedDuration, resourceMeter.totalTimeInSeconds, resourceMeter.totalMemoryUsageInMB, referenceGraph)
cleanup:
cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR)
where:
- totalBooks || expectedDuration | memoryLimit | referenceGraph
- 800 || 0.31 | 50 | false
- 1600 || 0.8 | 100 | false
- 3200 || 2.2 | 150 | false
- 6400 || 6.9 | 200 | true
+ totalBooks || expectedDuration | referenceGraph
+ 800 || 0.31 | false
+ 1600 || 0.8 | false
+ 3200 || 2.2 | false
+ 6400 || 6.9 | true
}
def 'Writing openroadm list data using saveListElements.'() {
cpsDataService.saveListElements(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, '/openroadm-devices', jsonListData, OffsetDateTime.now(), ContentType.JSON)
resourceMeter.stop()
then: 'the operation takes less than #expectedDuration with a margin of 100%'
- recordAndAssertResourceUsage("CPS:Saving list of ${totalNodes} devices", "Saving list of ${totalNodes} devices", expectedDuration, resourceMeter.totalTimeInSeconds, memoryLimit, resourceMeter.totalMemoryUsageInMB)
+ recordAndAssertResourceUsage("CPS:Saving list of ${totalNodes} devices", expectedDuration, resourceMeter.totalTimeInSeconds, resourceMeter.totalMemoryUsageInMB)
cleanup:
cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR)
where:
- totalNodes || expectedDuration | memoryLimit
- 50 || 1.5 | 100
- 100 || 3.0 | 200
- 200 || 6.4 | 400
- 400 || 14.0 | 500
+ totalNodes || expectedDuration
+ 50 || 1.5
+ 100 || 3.0
+ 200 || 6.4
+ 400 || 14.0
}
}
matches.size() == numberOfFiltersPerCmHandle * numberOfCmHandlesPerCmDataSubscription
and: 'query all subscribers within 1 second'
def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
- recordAndAssertResourceUsage('NCMP:Query all subscribers', "Query all subscribers", 1, durationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('NCMP:Query all subscribers', 1, durationInSeconds, resourceMeter.getTotalMemoryUsageInMB())
}
def 'Worst case subscription update (200x10 matching entries).'() {
def resultAfter = objectUnderTest.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_ANCHOR, cpsPath, INCLUDE_ALL_DESCENDANTS)
assert resultAfter.collect {it.leaves.subscribers.size()}.sum() == totalNumberOfEntries * (1 + numberOfCmDataSubscribers)
and: 'update matching subscription within 15 seconds'
- recordAndAssertResourceUsage('NCMP:Update matching subscription', "Update matching subscription", 10.5, durationInSeconds, 1000, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH)
+ recordAndAssertResourceUsage('NCMP:Update matching subscription', 10.5, durationInSeconds, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH)
}
def 'Worst case new subscription (200x10 new entries).'() {
resourceMeter.stop()
def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'insert new subscription with 1 second'
- recordAndAssertResourceUsage('NCMP:Insert new subscription', "Insert new subscription", 1.28, durationInSeconds, 100, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('NCMP:Insert new subscription', 1.02, durationInSeconds, resourceMeter.getTotalMemoryUsageInMB())
}
def querySubscriptionsByIteration(Collection<DataNode> allSubscriptionsAsDataNodes, targetSubscriptionSequenceNumber) {
}
resourceMeter.stop()
then: 'resource usage is as expected'
- recordAndAssertResourceUsage('NCMP:JVM warmup for CmHandleQueryPerfTest', 'JVM warmup for CmHandleQueryPerfTest', 60, resourceMeter.totalTimeInSeconds, 300, resourceMeter.totalMemoryUsageInMB)
+ recordAndAssertResourceUsage('NCMP:JVM warmup for CmHandleQueryPerfTest', 42, resourceMeter.totalTimeInSeconds, resourceMeter.totalMemoryUsageInMB)
}
def 'Query CM Handle IDs by a property name and value.'() {
resourceMeter.stop()
def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'the required operations are performed within required time'
- recordAndAssertResourceUsage('NCMP:CpsPath Registry attributes Query', "CpsPath Registry attributes Query", 3, durationInSeconds, 400, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH)
+ recordAndAssertResourceUsage('NCMP:CpsPath Registry attributes Query', 3, durationInSeconds, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH)
and: 'all nodes are returned'
result.size() == TOTAL_CM_HANDLES
and: 'the tree contains all the expected descendants too'
then: '100 data node have been found'
assert count == 100
and: 'average performance is as expected'
- recordAndAssertResourceUsage('NCMP:Look up CM-handle by id', 'Look up CM-handle by id', 0.58, resourceMeter.totalTimeInSeconds, 15, resourceMeter.totalMemoryUsageInMB)
+ recordAndAssertResourceUsage('NCMP:Look up CM-handle by id', 0.58, resourceMeter.totalTimeInSeconds, resourceMeter.totalMemoryUsageInMB)
}
def 'CM-handle is looked up by alternate id.'() {
then: 'all alternate ids are resolved correctly'
assert count == 100
and: 'average performance is as expected'
- recordAndAssertResourceUsage('NCMP:Look up CM-handle by alternate-id', 'Look up CM-handle by alternate-id', 1.75, resourceMeter.totalTimeInSeconds, 15, resourceMeter.totalMemoryUsageInMB, REFERENCE_GRAPH)
+ recordAndAssertResourceUsage('NCMP:Look up CM-handle by alternate-id', 1.6, resourceMeter.totalTimeInSeconds, resourceMeter.totalMemoryUsageInMB, REFERENCE_GRAPH)
}
def 'A batch of CM-handles is looked up by alternate id.'() {
then: 'expected amount of data was returned'
assert count == 100
then: 'average performance is as expected'
- recordAndAssertResourceUsage('NCMP:Batch look up CM-handle by alternate-id', 'Batch look up CM-handle by alternate-id', 0.4, resourceMeter.totalTimeInSeconds, 15, resourceMeter.totalMemoryUsageInMB)
+ recordAndAssertResourceUsage('NCMP:Batch look up CM-handle by alternate-id', 0.24, resourceMeter.totalTimeInSeconds, resourceMeter.totalMemoryUsageInMB)
}
def 'Find any CM-handle given moduleSetTag when there are 20K READY handles with same moduleSetTag.'() {
assert count == TOTAL_CM_HANDLES * 100
then: 'average performance is as expected'
def averageResponseTime = resourceMeter.totalTimeInSeconds / 100
- recordAndAssertResourceUsage('NCMP:Look up CM-handles by module-set-tag', 'Look up CM-handles by module-set-tag', 0.25, averageResponseTime, 500, resourceMeter.totalMemoryUsageInMB)
+ recordAndAssertResourceUsage('NCMP:Look up CM-handles by module-set-tag', 0.26, averageResponseTime, resourceMeter.totalMemoryUsageInMB)
}
}
Code Review / cps.git / commitdiff