addAnchorsWithData(OPENROADM_ANCHORS, CPS_PERFORMANCE_TEST_DATASPACE, LARGE_SCHEMA_SET, 'openroadm', data, ContentType.JSON)
resourceMeter.stop()
def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
- recordAndAssertResourceUsage('Creating openroadm anchors with large data tree', 100, durationInSeconds, 600, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Creating openroadm anchors with large data tree', 'Creating openroadm anchors with large data tree', 100, durationInSeconds, 600, resourceMeter.getTotalMemoryUsageInMB(), false)
}
def generateOpenRoadData(numberOfNodes) {
then: 'expected data exists'
assert result.xpath == ['/openroadm-devices']
then: 'operation completes within expected time'
- recordAndAssertResourceUsage('CPS pre-load test data',
- 100, resourceMeter.totalTimeInSeconds,
- 600, resourceMeter.totalMemoryUsageInMB)
+ recordAndAssertResourceUsage('CPS:Preload test data', 'CPS pre-load test data', 100, resourceMeter.totalTimeInSeconds, 600, resourceMeter.totalMemoryUsageInMB)
}
}
then: 'expected data exists'
assert result.xpath == [REGISTRY_PARENT]
and: 'operation completes within expected time'
- recordAndAssertResourceUsage('NCMP pre-load test data',
- 15, resourceMeter.totalTimeInSeconds,
- 600, resourceMeter.totalMemoryUsageInMB)
+ recordAndAssertResourceUsage('NCMP:Pre-load test data', 'NCMP pre-load test data', 15, resourceMeter.totalTimeInSeconds, 600, resourceMeter.totalMemoryUsageInMB)
}
}
import org.onap.cps.integration.base.CpsIntegrationSpecBase
+import java.text.DecimalFormat
+
abstract class PerfTestBase extends CpsIntegrationSpecBase {
static def LARGE_SCHEMA_SET = 'largeSchemaSet'
- static def PERFORMANCE_RECORD = []
+ static def PERFORMANCE_RECORDS = []
+ static def PERFORMANCE_CSV_RECORDS = []
static def DEFAULT_TIME_LIMIT_FACTOR = 2 // Allow 100% margin on top of expected (average) value
static def VERY_FAST_TEST_THRESHOLD = 0.01 // Definition of a very vast test (hard to measure)
static def DEFAULT_TIME_LIMIT_FACTOR_FOR_VERY_FAST_TEST = 3 // Allow 200% margin on very fast test (accuracy is an issue)
+ static def REFERENCE_GRAPH = true
+ static def CSV_PREFIX = "### CSV ### " // To be used in plot-job to extract CSV data from logs
+ static def TIME_FORMAT = new DecimalFormat("0.####")
+
def cleanupSpec() {
- println('##################################################################################################')
- printTitle()
- println('##################################################################################################')
- PERFORMANCE_RECORD.sort().each {
- println(it)
- }
- println('##################################################################################################')
- PERFORMANCE_RECORD.clear()
+ printTextRecords()
+ printCsvRecordsWithPrefix()
}
def setup() {
abstract def createInitialData()
- def recordAndAssertResourceUsage(shortTitle, double expectedAverageTimeInSec, double recordedTimeInSec, memoryLimitCurrentlyNotAsserted, double memoryUsageInMB, double timeLimitFactor) {
+ def recordAndAssertResourceUsage(String title, String shortTitle, double expectedAverageTimeInSec, double recordedTimeInSec, memoryLimitCurrentlyNotAsserted, double memoryUsageInMB, boolean referenceGraph) {
+ def timeLimitFactor = DEFAULT_TIME_LIMIT_FACTOR
if (expectedAverageTimeInSec <= VERY_FAST_TEST_THRESHOLD) {
timeLimitFactor = DEFAULT_TIME_LIMIT_FACTOR_FOR_VERY_FAST_TEST
}
def testPassed = recordedTimeInSec <= timeLimitFactor * expectedAverageTimeInSec
+ addRecord(shortTitle, expectedAverageTimeInSec, recordedTimeInSec, memoryUsageInMB, testPassed)
+ if (referenceGraph) {
+ addCsvRecord('Reference', title, expectedAverageTimeInSec, recordedTimeInSec)
+ }
+ addCsvRecord('All', title, expectedAverageTimeInSec, recordedTimeInSec)
+ assert recordedTimeInSec <= timeLimitFactor * expectedAverageTimeInSec
+ return true
+ }
+
+ def recordAndAssertResourceUsage(String title, String shortTitle, double expectedAverageTimeInSec, double recordedTimeInSec, memoryLimitCurrentlyNotAsserted, double memoryUsageInMB) {
+ recordAndAssertResourceUsage(title, shortTitle, expectedAverageTimeInSec, recordedTimeInSec, memoryLimitCurrentlyNotAsserted, memoryUsageInMB, false)
+ }
+
+ def addRecord(shortTitle, double expectedAverageTimeInSec, double recordedTimeInSec, double memoryUsageInMB, boolean testPassed) {
if (shortTitle.length() > 40) {
shortTitle = shortTitle.substring(0, 40)
}
- def record = String.format('%2d.%-40s limit %8.3f took %8.3f sec %,8.2f MB used ', PERFORMANCE_RECORD.size() + 1, shortTitle, expectedAverageTimeInSec, recordedTimeInSec, memoryUsageInMB)
+ def record = String.format('%2d.%-40s limit %8.3f took %8.3f sec %,8.2f MB used ', PERFORMANCE_RECORDS.size() + 1, shortTitle, expectedAverageTimeInSec, recordedTimeInSec, memoryUsageInMB)
record += testPassed ? 'PASS' : 'FAIL'
- PERFORMANCE_RECORD.add(record)
- assert recordedTimeInSec <= timeLimitFactor * expectedAverageTimeInSec
- return true
+ PERFORMANCE_RECORDS.add(record)
}
- def recordAndAssertResourceUsage(String shortTitle, double thresholdInSec, double recordedTimeInSec, memoryLimit, memoryUsageInMB) {
- recordAndAssertResourceUsage(shortTitle, thresholdInSec, recordedTimeInSec, memoryLimit, memoryUsageInMB, DEFAULT_TIME_LIMIT_FACTOR)
+
+
+ def addCsvRecord(tabName, title, double expectedAverageTimeInSec, double recordedTimeInSec) {
+ def csvRecord = String.format('%s,%s,%s,%s', tabName, title, TIME_FORMAT.format(expectedAverageTimeInSec), TIME_FORMAT.format(recordedTimeInSec))
+ PERFORMANCE_CSV_RECORDS.add(csvRecord)
}
+
+ def printTextRecords() {
+ println('##################################################################################################')
+ printTitle()
+ println('##################################################################################################')
+ PERFORMANCE_RECORDS.sort().each {
+ println(it)
+ }
+ PERFORMANCE_RECORDS.clear()
+ }
+
+ def printCsvRecordsWithPrefix() {
+ println('##################################### C S V F O R M A T ####################################')
+ PERFORMANCE_CSV_RECORDS.sort().each {
+ println CSV_PREFIX + it
+ }
+ PERFORMANCE_CSV_RECORDS.clear()
+ println('##################################################################################################')
+ }
+
}
}
resourceMeter.stop()
def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
- then: 'the operation completes within 12 seconds'
- recordAndAssertResourceUsage("Creating 33,000 books", 16, durationInSeconds, 150, resourceMeter.getTotalMemoryUsageInMB())
+ then: 'the operation completes within 16 seconds'
+ recordAndAssertResourceUsage('CPS:Creating 33,000 books', "Creating 33,000 books", 16, durationInSeconds, 150, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH)
}
def 'Get data nodes from multiple xpaths 32K (2^15) limit exceeded.'() {
resourceMeter.stop()
def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'test data is deleted in 1 second'
- recordAndAssertResourceUsage("Deleting test data", 0.068, durationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Deleting test data', "Deleting test data", 0.068, durationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB())
}
def countDataNodes() {
stopWatch.stop()
assert newModuleReferences.size() > 0 && newModuleReferences.size() < 300
}
- recordAndAssertResourceUsage('NCMP: Create large schema set', 2, stopWatch.getTotalTimeSeconds(), 3, 0)
+ recordAndAssertResourceUsage('CPS:Create large schema set', 'Create large schema set', 2, stopWatch.getTotalTimeSeconds(), 3, 0, REFERENCE_GRAPH)
}
def createModuleReferencesWithRandomMatchingExistingModuleReferences() {
resourceMeter.stop()
def setupDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'setup duration is below accepted margin of the expected average'
- recordAndAssertResourceUsage('Delete test setup', 103, setupDurationInSeconds, 800, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Delete test setup', 'Delete test setup', 100, setupDurationInSeconds, 800, resourceMeter.getTotalMemoryUsageInMB())
}
def 'Delete 100 container nodes'() {
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'delete duration is below accepted margin of the expected average'
- recordAndAssertResourceUsage('Delete 100 containers', 2.1, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Delete 100 containers', 'Delete 100 containers', 2.1, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB())
}
def 'Batch delete 100 container nodes'() {
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'delete duration is below accepted margin of the expected average'
- recordAndAssertResourceUsage('Batch delete 100 containers', 0.65, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Batch delete 100 containers', 'Batch delete 100 containers', 0.65, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB())
}
def 'Delete 100 list elements'() {
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'delete duration is below accepted margin of the expected average'
- recordAndAssertResourceUsage('Delete 100 lists elements', 2.3, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Delete 100 lists elements', 'Delete 100 lists elements', 2.3, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH)
}
def 'Batch delete 100 list elements'() {
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'delete duration is below accepted margin of the expected average'
- recordAndAssertResourceUsage('Batch delete 100 lists elements', 0.7, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Batch delete 100 lists elements', 'Batch delete 100 lists elements', 0.7, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB())
}
def 'Delete 100 whole lists'() {
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'delete duration is below accepted margin (4x) of the expected average'
- recordAndAssertResourceUsage('Delete 100 whole lists', 3.2, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB(), 4)
+ recordAndAssertResourceUsage('CPS:Delete 100 whole lists', 'Delete 100 whole lists', 3.2, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH)
}
def 'Batch delete 100 whole lists'() {
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'delete duration is below accepted margin (3x) of the expected average'
- recordAndAssertResourceUsage('Batch delete 100 whole lists', 2.4, deleteDurationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB(), 4)
+ recordAndAssertResourceUsage('CPS:Batch delete 100 whole lists', 'Batch delete 100 whole lists', 2.4, deleteDurationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH)
}
def 'Delete 1 large data node'() {
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'delete duration is below accepted margin of the expected average'
- recordAndAssertResourceUsage('Delete one large node', 1.9, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Delete one large node', 'Delete one large node', 1.9, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
}
def 'Delete root node with many descendants'() {
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'delete duration is below accepted margin of the expected average'
- recordAndAssertResourceUsage('Delete root node', 1.9, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Delete root node', 'Delete root node', 1.9, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
}
def 'Delete data nodes for an anchor'() {
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'delete duration is below accepted margin of the expected average'
- recordAndAssertResourceUsage('Delete data nodes for anchor', 1.9, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Delete data nodes for anchor', 'Delete data nodes for anchor', 1.9, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
}
def 'Batch delete 100 non-existing nodes'() {
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'delete duration is below accepted margin of the expected average'
- recordAndAssertResourceUsage('Batch delete 100 non-existing', 2, deleteDurationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB(),3)
+ recordAndAssertResourceUsage('CPS:Batch delete 100 non-existing data nodes', 'Batch delete 100 non-existing', 2, deleteDurationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB())
}
def 'Clean up test data'() {
resourceMeter.stop()
def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'delete duration is below accepted margin of the expected average'
- recordAndAssertResourceUsage('Delete test cleanup', 11.09, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Delete test cleanup', 'Delete test cleanup', 10, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
}
}
assert countDataNodesInTree(result) == expectedNumberOfDataNodes
def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'all data is read within #expectedDuration seconds and a margin of 100%'
- recordAndAssertResourceUsage("Read datatrees with ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage("CPS:Read datatree with ${scenario}", "Read datatrees with ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB(), referencegraph)
where: 'the following parameters are used'
- scenario | fetchDescendantsOption || expectedDuration | memoryLimit | expectedNumberOfDataNodes
- 'no descendants' | OMIT_DESCENDANTS || 0.01 | 1 | 1
- 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.045 | 5 | 1 + OPENROADM_DEVICES_PER_ANCHOR
- 'all descendants' | INCLUDE_ALL_DESCENDANTS || 1.1 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
+ scenario | fetchDescendantsOption | referencegraph || expectedDuration | memoryLimit | expectedNumberOfDataNodes
+ 'no descendants' | OMIT_DESCENDANTS | false || 0.01 | 1 | 1
+ 'direct descendants' | DIRECT_CHILDREN_ONLY | false || 0.045 | 5 | 1 + OPENROADM_DEVICES_PER_ANCHOR
+ 'all descendants' | INCLUDE_ALL_DESCENDANTS | true || 1.1 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
}
def 'Read data trees for multiple xpaths'() {
then: 'requested nodes and their descendants are returned'
assert countDataNodesInTree(result) == OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
and: 'all data is read within expected time and a margin of 200%'
- recordAndAssertResourceUsage("Read datatrees for multiple xpaths", 2.0, durationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB(),3)
+ recordAndAssertResourceUsage('CPS:Read datatrees for multiple xpaths', "Read datatrees for multiple xpaths", 2.0, durationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB())
}
def 'Read for multiple xpaths to non-existing datanodes'() {
then: 'no data is returned'
assert result.isEmpty()
and: 'the operation completes within within expected time'
- recordAndAssertResourceUsage("Read non-existing xpaths", 0.06, durationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Read non-existing xpaths', "Read non-existing xpaths", 0.06, durationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB())
}
def 'Read complete data trees using #scenario.'() {
resourceMeter.stop()
def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'all data is read within expected time and a margin of 100%'
- recordAndAssertResourceUsage("Read datatrees using ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB(),3)
+ recordAndAssertResourceUsage("CPS:Read datatrees using ${scenario}", "Read datatrees using ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
where: 'the following xpaths are used'
scenario | xpath || expectedDuration | memoryLimit | expectedNumberOfDataNodes
'openroadm root' | '/' || 1 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
}
resourceMeter.stop()
then: 'operation takes less than expected duration'
- recordAndAssertResourceUsage('Module query test setup', 90, resourceMeter.totalTimeInSeconds, 500, resourceMeter.totalMemoryUsageInMB
- )
+ recordAndAssertResourceUsage('CPS:Module query test setup', 'Module query test setup', 150, resourceMeter.totalTimeInSeconds, 500, resourceMeter.totalMemoryUsageInMB)
}
def 'Querying anchors by module name is NOT dependant on the file size of the module.'() {
then: 'expected number of anchors is returned'
assert result.size() == TOTAL_TEST_ANCHORS
and: 'operation completes with expected resource usage'
- recordAndAssertResourceUsage("Query for anchors with ${scenario}", 0.1, resourceMeter.totalTimeInSeconds, 5, resourceMeter.totalMemoryUsageInMB)
+ recordAndAssertResourceUsage("CPS:Query for anchors with ${scenario}", "Query for anchors with ${scenario}", 0.1, resourceMeter.totalTimeInSeconds, 5, resourceMeter.totalMemoryUsageInMB, REFERENCE_GRAPH)
where: 'the following parameters are used'
scenario | yangModuleName
'1 KB module' | 'module0'
then: 'the expected number of nodes is returned'
assert countDataNodesInTree(result) == expectedNumberOfDataNodes
and: 'all data is read #expectedDuration seconds with a margin of 100%'
- recordAndAssertResourceUsage("Query 1 anchor ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage("CPS:Query 1 anchor ${scenario}", "Query 1 anchor ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
where: 'the following parameters are used'
scenario | cpsPath || expectedDuration | memoryLimit | expectedNumberOfDataNodes
'top element' | '/openroadm-devices' || 0.89 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1
then: 'the expected number of nodes is returned'
assert countDataNodesInTree(result) == expectedNumberOfDataNodes
and: 'all data is read #expectedDuration seconds with a margin of 100%'
- recordAndAssertResourceUsage("Query across anchors ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage("CPS:Query across anchors ${scenario}", "Query across anchors ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH)
where: 'the following parameters are used'
scenario | cpspath || expectedDuration | memoryLimit | expectedNumberOfDataNodes
'top element' | '/openroadm-devices' || 3.0 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1)
then: 'the expected number of nodes is returned'
assert countDataNodesInTree(result) == expectedNumberOfDataNodes
and: 'all data is read #expectedDuration seconds with a margin of 100%'
- recordAndAssertResourceUsage("Query with ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB(),2)
+ recordAndAssertResourceUsage("CPS:Query with ${scenario}", "Query with ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
where: 'the following parameters are used'
scenario | fetchDescendantsOption || expectedDuration | memoryLimit | expectedNumberOfDataNodes
'no descendants' | OMIT_DESCENDANTS || 0.09 | 6 | OPENROADM_DEVICES_PER_ANCHOR
then: 'the expected number of nodes is returned'
assert countDataNodesInTree(result) == expectedNumberOfDataNodes
and: 'all data is read #expectedDuration seconds with a margin of 100%'
- recordAndAssertResourceUsage("Query ancestors with ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage("CPS:Query ancestors with ${scenario}", "Query ancestors with ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
where: 'the following parameters are used'
scenario | fetchDescendantsOption || expectedDuration | memoryLimit | expectedNumberOfDataNodes
'no descendants' | OMIT_DESCENDANTS || 0.08 | 3 | 1
then: 'the expected number of results is returned'
assert result.size() == expectedNumberOfValues
and: 'all data is read #expectedDuration seconds with a margin of 100%'
- recordAndAssertResourceUsage("Query data leaf ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage("CPS:Query data leaf ${scenario}", "Query data leaf ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
where: 'the following parameters are used'
scenario | cpsPath || expectedDuration | memoryLimit | expectedNumberOfValues
'unique leaf value' | '/openroadm-devices/openroadm-device/@device-id' || 0.05 | 0.1 | OPENROADM_DEVICES_PER_ANCHOR
assert totalNodes == countDataNodes(changeLeaves? '/openroadm-devices/openroadm-device[@status="fail"]'
: '/openroadm-devices/openroadm-device[@status="success"]')
and: 'update completes within expected time and a margin of 100%'
- recordAndAssertResourceUsage(scenario, expectedTime, resourceMeter.getTotalTimeInSeconds(), memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage("CPS:${scenario}", scenario, expectedTime, resourceMeter.getTotalTimeInSeconds(), memoryLimit, resourceMeter.getTotalMemoryUsageInMB(), referenceGraph)
where:
- scenario | totalNodes | startId | changeLeaves || expectedTime | memoryLimit
- 'Replace 0 nodes with 100' | 100 | 1 | false || 3.0 | 200
- 'Replace 100 using same data' | 100 | 1 | false || 5.5 | 200
- 'Replace 100 with new leaf values' | 100 | 1 | true || 5.4 | 200
- 'Replace 100 with 100 new nodes' | 100 | 101 | false || 9.3 | 200
- 'Replace 50 existing and 50 new' | 100 | 151 | true || 7.3 | 200
- 'Replace 100 nodes with 0' | 0 | 1 | false || 6.0 | 200
+ scenario | totalNodes | startId | changeLeaves || expectedTime | memoryLimit | referenceGraph
+ 'Replace 0 nodes with 100' | 100 | 1 | false || 3.0 | 200 | false
+ 'Replace 100 using same data' | 100 | 1 | false || 5.5 | 200 | false
+ 'Replace 100 with new leaf values' | 100 | 1 | true || 5.4 | 200 | false
+ 'Replace 100 with 100 new nodes' | 100 | 101 | false || 9.3 | 200 | true
+ 'Replace 50 existing and 50 new' | 100 | 151 | true || 7.3 | 200 | false
+ 'Replace 100 nodes with 0' | 0 | 1 | false || 6.0 | 200 | false
}
def 'Replace list content: #scenario.'() {
assert totalNodes == countDataNodes(changeLeaves? '/openroadm-devices/openroadm-device[@status="fail"]'
: '/openroadm-devices/openroadm-device[@status="success"]')
and: 'update completes within expected time and a margin of 100%'
- recordAndAssertResourceUsage(scenario,
- expectedTime, resourceMeter.getTotalTimeInSeconds(),
- memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage("CPS:${scenario}", scenario, expectedTime, resourceMeter.getTotalTimeInSeconds(), memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
where:
scenario | totalNodes | startId | changeLeaves || expectedTime | memoryLimit
'Replace list of 0 with 100' | 100 | 1 | false || 3.0 | 200
then: 'data leaves have expected values'
assert 100 == countDataNodes('/openroadm-devices/openroadm-device[@status="fail"]')
and: 'update completes within expected time and a margin of 100%'
- recordAndAssertResourceUsage('Update leaves for 100 data nodes', 0.22, resourceMeter.getTotalTimeInSeconds(), 120, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('CPS:Update leaves for 100 data nodes', 'Update leaves for 100 data nodes', 0.22, resourceMeter.getTotalTimeInSeconds(), 120, resourceMeter.getTotalMemoryUsageInMB())
}
def 'Clean up for CPS Update API.'() {
cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, jsonData, OffsetDateTime.now())
resourceMeter.stop()
then: 'the operation takes less than #expectedDuration with a margin of 100%'
- recordAndAssertResourceUsage("Writing ${totalNodes} devices", expectedDuration, resourceMeter.getTotalTimeInSeconds(), memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage("CPS:Writing ${totalNodes} devices", "Writing ${totalNodes} devices", expectedDuration, resourceMeter.getTotalTimeInSeconds(), memoryLimit, resourceMeter.getTotalMemoryUsageInMB(), referenceGraph)
cleanup:
cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR)
where:
- totalNodes || expectedDuration | memoryLimit
- 50 || 1.45 | 100
- 100 || 2.9 | 200
- 200 || 6.2 | 400
- 400 || 13.0 | 500
+ totalNodes || expectedDuration | memoryLimit | referenceGraph
+ 50 || 1.45 | 100 | false
+ 100 || 2.9 | 200 | false
+ 200 || 6.2 | 400 | true
+ 400 || 13.0 | 500 | false
}
def 'Writing bookstore data has exponential time.'() {
cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, '/bookstore/categories[@code=1]', booksData, OffsetDateTime.now())
resourceMeter.stop()
then: 'the operation takes less than #expectedDuration with a margin of 100%'
- recordAndAssertResourceUsage("Writing ${totalBooks} books",
- expectedDuration, resourceMeter.totalTimeInSeconds,
- memoryLimit, resourceMeter.totalMemoryUsageInMB)
+ recordAndAssertResourceUsage("CPS:Writing ${totalBooks} books", "Writing ${totalBooks} books", expectedDuration, resourceMeter.totalTimeInSeconds, memoryLimit, resourceMeter.totalMemoryUsageInMB, referenceGraph)
cleanup:
cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR)
where:
- totalBooks || expectedDuration | memoryLimit
- 800 || 0.31 | 50
- 1600 || 0.8 | 100
- 3200 || 2.2 | 150
- 6400 || 6.9 | 200
+ totalBooks || expectedDuration | memoryLimit | referenceGraph
+ 800 || 0.31 | 50 | false
+ 1600 || 0.8 | 100 | false
+ 3200 || 2.2 | 150 | false
+ 6400 || 6.9 | 200 | true
}
def 'Writing openroadm list data using saveListElements.'() {
cpsDataService.saveListElements(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, '/openroadm-devices', jsonListData, OffsetDateTime.now(), ContentType.JSON)
resourceMeter.stop()
then: 'the operation takes less than #expectedDuration with a margin of 100%'
- recordAndAssertResourceUsage("Saving list of ${totalNodes} devices", expectedDuration, resourceMeter.totalTimeInSeconds, memoryLimit, resourceMeter.totalMemoryUsageInMB)
+ recordAndAssertResourceUsage("CPS:Saving list of ${totalNodes} devices", "Saving list of ${totalNodes} devices", expectedDuration, resourceMeter.totalTimeInSeconds, memoryLimit, resourceMeter.totalMemoryUsageInMB)
cleanup:
cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR)
where:
}
def 'Alternate Id Lookup Performance.'() {
- when: 'perform a 1,000 lookups by alternate id'
+ when: 'perform a 10 lookups by alternate id' // Increase to 1,000 for more accurate result while tuning
resourceMeter.start()
- (1..1000).each {
+ (1..10).each {
networkCmProxyInventoryFacade.getNcmpServiceCmHandle("${altIdPrefix}alt=${it}")
}
resourceMeter.stop()
matches.size() == numberOfFiltersPerCmHandle * numberOfCmHandlesPerCmDataSubscription
and: 'query all subscribers within 1 second'
def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
- recordAndAssertResourceUsage("Query all subscribers", 2.56, durationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('NCMP:Query all subscribers', "Query all subscribers", 1, durationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB())
}
def 'Worst case subscription update (200x10 matching entries).'() {
def resultAfter = objectUnderTest.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_ANCHOR, cpsPath, INCLUDE_ALL_DESCENDANTS)
assert resultAfter.collect {it.leaves.subscribers.size()}.sum() == totalNumberOfEntries * (1 + numberOfCmDataSubscribers)
and: 'update matching subscription within 15 seconds'
- recordAndAssertResourceUsage("Update matching subscription", 10.5, durationInSeconds, 1000, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('NCMP:Update matching subscription', "Update matching subscription", 10.5, durationInSeconds, 1000, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH)
}
def 'Worst case new subscription (200x10 new entries).'() {
resourceMeter.stop()
def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'insert new subscription with 1 second'
- recordAndAssertResourceUsage("Insert new subscription", 1.28, durationInSeconds, 100, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('NCMP:Insert new subscription', "Insert new subscription", 1.28, durationInSeconds, 100, resourceMeter.getTotalMemoryUsageInMB())
}
def querySubscriptionsByIteration(Collection<DataNode> allSubscriptionsAsDataNodes, targetSubscriptionSequenceNumber) {
}
resourceMeter.stop()
then: 'resource usage is as expected'
- recordAndAssertResourceUsage('JVM warmup for CmHandleQueryPerfTest', 60, resourceMeter.totalTimeInSeconds, 300, resourceMeter.totalMemoryUsageInMB)
+ recordAndAssertResourceUsage('NCMP:JVM warmup for CmHandleQueryPerfTest', 'JVM warmup for CmHandleQueryPerfTest', 60, resourceMeter.totalTimeInSeconds, 300, resourceMeter.totalMemoryUsageInMB)
}
def 'Query CM Handle IDs by a property name and value.'() {
resourceMeter.stop()
def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
then: 'the required operations are performed within required time'
- recordAndAssertResourceUsage("CpsPath Registry attributes Query", 3, durationInSeconds, 400, resourceMeter.getTotalMemoryUsageInMB())
+ recordAndAssertResourceUsage('NCMP:CpsPath Registry attributes Query', "CpsPath Registry attributes Query", 3, durationInSeconds, 400, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH)
and: 'all nodes are returned'
result.size() == TOTAL_CM_HANDLES
and: 'the tree contains all the expected descendants too'
then: '100 data node have been found'
assert count == 100
and: 'average performance is as expected'
- recordAndAssertResourceUsage('Look up CM-handle by id', 0.58, resourceMeter.totalTimeInSeconds, 15, resourceMeter.totalMemoryUsageInMB)
+ recordAndAssertResourceUsage('NCMP:Look up CM-handle by id', 'Look up CM-handle by id', 0.58, resourceMeter.totalTimeInSeconds, 15, resourceMeter.totalMemoryUsageInMB)
}
def 'CM-handle is looked up by alternate id.'() {
then: 'all alternate ids are resolved correctly'
assert count == 100
and: 'average performance is as expected'
- recordAndAssertResourceUsage('Look up CM-handle by alternate-id', 1.75, resourceMeter.totalTimeInSeconds, 15, resourceMeter.totalMemoryUsageInMB)
+ recordAndAssertResourceUsage('NCMP:Look up CM-handle by alternate-id', 'Look up CM-handle by alternate-id', 1.75, resourceMeter.totalTimeInSeconds, 15, resourceMeter.totalMemoryUsageInMB, REFERENCE_GRAPH)
}
def 'A batch of CM-handles is looked up by alternate id.'() {
then: 'expected amount of data was returned'
assert count == 100
then: 'average performance is as expected'
- recordAndAssertResourceUsage('Batch look up CM-handle by alternate-id', 0.4, resourceMeter.totalTimeInSeconds, 15, resourceMeter.totalMemoryUsageInMB)
+ recordAndAssertResourceUsage('NCMP:Batch look up CM-handle by alternate-id', 'Batch look up CM-handle by alternate-id', 0.4, resourceMeter.totalTimeInSeconds, 15, resourceMeter.totalMemoryUsageInMB)
}
def 'Find any CM-handle given moduleSetTag when there are 20K READY handles with same moduleSetTag.'() {
assert count == TOTAL_CM_HANDLES * 100
then: 'average performance is as expected'
def averageResponseTime = resourceMeter.totalTimeInSeconds / 100
- recordAndAssertResourceUsage('Look up CM-handles by module-set-tag', 0.25, averageResponseTime, 500, resourceMeter.totalMemoryUsageInMB)
+ recordAndAssertResourceUsage('NCMP:Look up CM-handles by module-set-tag', 'Look up CM-handles by module-set-tag', 0.25, averageResponseTime, 500, resourceMeter.totalMemoryUsageInMB)
}
}
Code Review / cps.git / commitdiff