Make Groovy Performance Test Stable #6

- Fixed many expectations again now Groovy Perf does not run at same time as KPI
  (strangely enough most create use cases got worse)
- Default margin now 100% instead of 50%
- Increased volume of 1 very fast test to get more stable measurements

Issue-ID:CPS-2743

Change-Id: I081d48926a9e283a429603bd78dcd79c5d8a2394
Signed-off-by: ToineSiebelink <toine.siebelink@est.tech>

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy
index a617748..cc83bf6 100644 (file)
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy
@@ -26,7 +26,7 @@ abstract class PerfTestBase extends CpsIntegrationSpecBase {
 
     static def LARGE_SCHEMA_SET = 'largeSchemaSet'
     static def PERFORMANCE_RECORD = []
-    static def DEFAULT_TIME_LIMIT_FACTOR = 1.5                  // Allow 50% margin on top of expected (average) value
+    static def DEFAULT_TIME_LIMIT_FACTOR = 2                    // Allow 100% margin on top of expected (average) value
     static def VERY_FAST_TEST_THRESHOLD = 0.01                  // Definition of a very vast test (hard to measure)
     static def DEFAULT_TIME_LIMIT_FACTOR_FOR_VERY_FAST_TEST = 3 // Allow 200% margin on very fast test (accuracy is an issue)
 

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy
index da34fe5..e242a4c 100644 (file)
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy
@@ -47,7 +47,7 @@ class CpsDataServiceLimitsPerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'the operation completes within 12 seconds'
-            recordAndAssertResourceUsage("Creating 33,000 books", 12, durationInSeconds, 150, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage("Creating 33,000 books", 16, durationInSeconds, 150, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Get data nodes from multiple xpaths 32K (2^15) limit exceeded.'() {
@@ -88,7 +88,7 @@ class CpsDataServiceLimitsPerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'test data is deleted in 1 second'
-            recordAndAssertResourceUsage("Deleting test data", 0.056, durationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage("Deleting test data", 0.068, durationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def countDataNodes() {

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy
index be9252a..ad98d30 100644 (file)
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy
@@ -57,7 +57,7 @@ class DeletePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'delete duration is below accepted margin of the expected average'
-            recordAndAssertResourceUsage('Delete 100 containers', 1.4, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage('Delete 100 containers', 2.1, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Batch delete 100 container nodes'() {
@@ -87,7 +87,7 @@ class DeletePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'delete duration is below accepted margin of the expected average'
-            recordAndAssertResourceUsage('Delete 100 lists elements', 1.5, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage('Delete 100 lists elements', 2.3, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Batch delete 100 list elements'() {
@@ -129,7 +129,7 @@ class DeletePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'delete duration is below accepted margin (3x) of the expected average'
-            recordAndAssertResourceUsage('Batch delete 100 whole lists', 3.8, deleteDurationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB(), 4)
+            recordAndAssertResourceUsage('Batch delete 100 whole lists', 3.0, deleteDurationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB(), 4)
     }
 
     def 'Delete 1 large data node'() {
@@ -139,7 +139,7 @@ class DeletePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'delete duration is below accepted margin of the expected average'
-            recordAndAssertResourceUsage('Delete one large node', 2, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage('Delete one large node', 1.9, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Delete root node with many descendants'() {
@@ -149,7 +149,7 @@ class DeletePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'delete duration is below accepted margin of the expected average'
-            recordAndAssertResourceUsage('Delete root node', 2, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage('Delete root node', 1.9, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Delete data nodes for an anchor'() {
@@ -159,7 +159,7 @@ class DeletePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'delete duration is below accepted margin of the expected average'
-            recordAndAssertResourceUsage('Delete data nodes for anchor', 2, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage('Delete data nodes for anchor', 1.8, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Batch delete 100 non-existing nodes'() {

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy
index aa92b39..68c29b2 100644 (file)
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy
@@ -40,12 +40,12 @@ class GetPerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             assert countDataNodesInTree(result) == expectedNumberOfDataNodes
             def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
-        then: 'all data is read within #expectedDuration seconds and a margin of 50%'
+        then: 'all data is read within #expectedDuration seconds and a margin of 100%'
             recordAndAssertResourceUsage("Read datatrees with ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
         where: 'the following parameters are used'
             scenario             | fetchDescendantsOption  || expectedDuration | memoryLimit | expectedNumberOfDataNodes
             'no descendants'     | OMIT_DESCENDANTS        || 0.01             | 1           | 1
-            'direct descendants' | DIRECT_CHILDREN_ONLY    || 0.03             | 5           | 1 + OPENROADM_DEVICES_PER_ANCHOR
+            'direct descendants' | DIRECT_CHILDREN_ONLY    || 0.05             | 5           | 1 + OPENROADM_DEVICES_PER_ANCHOR
             'all descendants'    | INCLUDE_ALL_DESCENDANTS || 1.1              | 250         | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
     }
 
@@ -60,12 +60,12 @@ class GetPerfTest extends CpsPerfTestBase {
         then: 'requested nodes and their descendants are returned'
             assert countDataNodesInTree(result) == OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
         and: 'all data is read within expected time and a margin of 200%'
-            recordAndAssertResourceUsage("Read datatrees for multiple xpaths", 1.3, durationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB(),3)
+            recordAndAssertResourceUsage("Read datatrees for multiple xpaths", 2.0, durationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB(),3)
     }
 
     def 'Read for multiple xpaths to non-existing datanodes'() {
         given: 'a collection of xpaths to get'
-            def xpaths = (1..50).collect { "/path/to/non-existing/node[@id='" + it + "']" }
+            def xpaths = (1..500).collect { "/path/to/non-existing/node[@id='" + it + "']" }
         when: 'get data nodes from 1 anchor'
             resourceMeter.start()
             def result = objectUnderTest.getDataNodesForMultipleXpaths(CPS_PERFORMANCE_TEST_DATASPACE, 'openroadm2', xpaths, INCLUDE_ALL_DESCENDANTS)
@@ -74,7 +74,7 @@ class GetPerfTest extends CpsPerfTestBase {
         then: 'no data is returned'
             assert result.isEmpty()
         and: 'the operation completes within within expected time'
-            recordAndAssertResourceUsage("Read non-existing xpaths", 0.01, durationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage("Read non-existing xpaths", 0.15, durationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Read complete data trees using #scenario.'() {
@@ -84,12 +84,12 @@ class GetPerfTest extends CpsPerfTestBase {
             assert countDataNodesInTree(result) == expectedNumberOfDataNodes
             resourceMeter.stop()
             def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
-        then: 'all data is read within expected time and a margin of 50%'
-            recordAndAssertResourceUsage("Read datatrees using ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+        then: 'all data is read within expected time and a margin of 100%'
+            recordAndAssertResourceUsage("Read datatrees using ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB(),3)
         where: 'the following xpaths are used'
             scenario                | xpath                                  || expectedDuration | memoryLimit | expectedNumberOfDataNodes
             'openroadm root'        | '/'                                    || 1                | 250         | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
-            'openroadm top element' | '/openroadm-devices'                   || 0.9              | 250         | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
+            'openroadm top element' | '/openroadm-devices'                   || 1.0              | 250         | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
             'openroadm whole list'  | '/openroadm-devices/openroadm-device'  || 1.06             | 250         | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
     }
 

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy
index 8c37835..625cc00 100644 (file)
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy
@@ -41,13 +41,13 @@ class QueryPerfTest extends CpsPerfTestBase {
             def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'the expected number of nodes is returned'
             assert countDataNodesInTree(result) == expectedNumberOfDataNodes
-        and: 'all data is read #expectedDuration seconds with a margin of 50%'
+        and: 'all data is read #expectedDuration seconds with a margin of 100%'
             recordAndAssertResourceUsage("Query 1 anchor ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
         where: 'the following parameters are used'
             scenario                     | cpsPath                                                             || expectedDuration | memoryLimit | expectedNumberOfDataNodes
             'top element'                | '/openroadm-devices'                                                || 0.89             | 400         | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1
             'leaf condition'             | '//openroadm-device[@ne-state="inservice"]'                         || 1.3              | 400         | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
-            'ancestors'                  | '//openroadm-device/ancestor::openroadm-devices'                    || 1.1              | 400         | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1
+            'ancestors'                  | '//openroadm-device/ancestor::openroadm-devices'                    || 1.3              | 400         | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1
             'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 0.93             | 400         | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1
             'non-existing data'          | '/path/to/non-existing/node[@id="1"]'                               || 0.01             | 1           | 0
     }
@@ -60,7 +60,7 @@ class QueryPerfTest extends CpsPerfTestBase {
             def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'the expected number of nodes is returned'
             assert countDataNodesInTree(result) == expectedNumberOfDataNodes
-        and: 'all data is read #expectedDuration seconds with a margin of 50%'
+        and: 'all data is read #expectedDuration seconds with a margin of 100%'
             recordAndAssertResourceUsage("Query across anchors ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
         where: 'the following parameters are used'
             scenario                     | cpspath                                                             || expectedDuration | memoryLimit | expectedNumberOfDataNodes
@@ -78,12 +78,12 @@ class QueryPerfTest extends CpsPerfTestBase {
             def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'the expected number of nodes is returned'
             assert countDataNodesInTree(result) == expectedNumberOfDataNodes
-        and: 'all data is read #expectedDuration seconds with a margin of 50%'
+        and: 'all data is read #expectedDuration seconds with a margin of 100%'
             recordAndAssertResourceUsage("Query with ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB(),2)
         where: 'the following parameters are used'
             scenario             | fetchDescendantsOption  || expectedDuration | memoryLimit | expectedNumberOfDataNodes
             'no descendants'     | OMIT_DESCENDANTS        || 0.08             | 6           | OPENROADM_DEVICES_PER_ANCHOR
-            'direct descendants' | DIRECT_CHILDREN_ONLY    || 0.09             | 12          | OPENROADM_DEVICES_PER_ANCHOR * 2
+            'direct descendants' | DIRECT_CHILDREN_ONLY    || 0.12             | 12          | OPENROADM_DEVICES_PER_ANCHOR * 2
             'all descendants'    | INCLUDE_ALL_DESCENDANTS || 1.05             | 200         | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
     }
 
@@ -95,12 +95,12 @@ class QueryPerfTest extends CpsPerfTestBase {
             def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'the expected number of nodes is returned'
             assert countDataNodesInTree(result) == expectedNumberOfDataNodes
-        and: 'all data is read #expectedDuration seconds with a margin of 50%'
+        and: 'all data is read #expectedDuration seconds with a margin of 100%'
             recordAndAssertResourceUsage("Query ancestors with ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
         where: 'the following parameters are used'
             scenario             | fetchDescendantsOption  || expectedDuration | memoryLimit | expectedNumberOfDataNodes
-            'no descendants'     | OMIT_DESCENDANTS        || 0.06             | 3           | 1
-            'direct descendants' | DIRECT_CHILDREN_ONLY    || 0.07             | 8           | 1 + OPENROADM_DEVICES_PER_ANCHOR
+            'no descendants'     | OMIT_DESCENDANTS        || 0.08             | 3           | 1
+            'direct descendants' | DIRECT_CHILDREN_ONLY    || 0.09             | 8           | 1 + OPENROADM_DEVICES_PER_ANCHOR
             'all descendants'    | INCLUDE_ALL_DESCENDANTS || 1.0              | 400         | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
     }
 
@@ -112,7 +112,7 @@ class QueryPerfTest extends CpsPerfTestBase {
             def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'the expected number of results is returned'
             assert result.size() == expectedNumberOfValues
-        and: 'all data is read #expectedDuration seconds with a margin of 50%'
+        and: 'all data is read #expectedDuration seconds with a margin of 100%'
             recordAndAssertResourceUsage("Query data leaf ${scenario}", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
         where: 'the following parameters are used'
             scenario                     | cpsPath                                             || expectedDuration | memoryLimit | expectedNumberOfValues

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy
index 0c66871..c2b027d 100644 (file)
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy
@@ -75,16 +75,16 @@ class UpdatePerfTest extends CpsPerfTestBase {
         and: 'data leaves have expected values'
             assert totalNodes == countDataNodes(changeLeaves? '/openroadm-devices/openroadm-device[@status="fail"]'
                                                             : '/openroadm-devices/openroadm-device[@status="success"]')
-        and: 'update completes within expected time and a margin of 50%'
+        and: 'update completes within expected time and a margin of 100%'
             recordAndAssertResourceUsage(scenario, expectedTime, resourceMeter.getTotalTimeInSeconds(), memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
         where:
             scenario                           | totalNodes | startId | changeLeaves || expectedTime | memoryLimit
-            'Replace 0 nodes with 100'         | 100        | 1       | false        || 2.85         | 200
-            'Replace 100 using same data'      | 100        | 1       | false        || 2.0          | 200
-            'Replace 100 with new leaf values' | 100        | 1       | true         || 2.0          | 200
-            'Replace 100 with 100 new nodes'   | 100        | 101     | false        || 6.2          | 200
-            'Replace 50 existing and 50 new'   | 100        | 151     | true         || 4.2          | 200
-            'Replace 100 nodes with 0'         | 0          | 1       | false        || 3.5          | 200
+            'Replace 0 nodes with 100'         | 100        | 1       | false        || 3.0          | 200
+            'Replace 100 using same data'      | 100        | 1       | false        || 5.0          | 200
+            'Replace 100 with new leaf values' | 100        | 1       | true         || 5.4          | 200
+            'Replace 100 with 100 new nodes'   | 100        | 101     | false        || 9.3          | 200
+            'Replace 50 existing and 50 new'   | 100        | 151     | true         || 7.3          | 200
+            'Replace 100 nodes with 0'         | 0          | 1       | false        || 6.0          | 200
     }
 
     def 'Replace list content: #scenario.'() {
@@ -99,18 +99,18 @@ class UpdatePerfTest extends CpsPerfTestBase {
         and: 'data leaves have expected values'
             assert totalNodes == countDataNodes(changeLeaves? '/openroadm-devices/openroadm-device[@status="fail"]'
                                                             : '/openroadm-devices/openroadm-device[@status="success"]')
-        and: 'update completes within expected time and a margin of 50%'
+        and: 'update completes within expected time and a margin of 100%'
             recordAndAssertResourceUsage(scenario,
                     expectedTime, resourceMeter.getTotalTimeInSeconds(),
                     memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
         where:
             scenario                                   | totalNodes | startId | changeLeaves || expectedTime | memoryLimit
             'Replace list of 0 with 100'               | 100        | 1       | false        || 3.0          | 200
-            'Replace list of 100 using same data'      | 100        | 1       | false        || 2.0          | 200
-            'Replace list of 100 with new leaf values' | 100        | 1       | true         || 2.0          | 200
-            'Replace list with 100 new nodes'          | 100        | 101     | false        || 6.1          | 200
-            'Replace list with 50 existing and 50 new' | 100        | 151     | true         || 4.0          | 200
-            'Replace list of 100 nodes with 1'         | 1          | 1       | false        || 3.34         | 200
+            'Replace list of 100 using same data'      | 100        | 1       | false        || 5.5          | 200
+            'Replace list of 100 with new leaf values' | 100        | 1       | true         || 5.5          | 200
+            'Replace list with 100 new nodes'          | 100        | 101     | false        || 9.0          | 200
+            'Replace list with 50 existing and 50 new' | 100        | 151     | true         || 7.0          | 200
+            'Replace list of 100 nodes with 1'         | 1          | 1       | false        || 5.9          | 200
     }
 
     def 'Update leaves for 100 data nodes.'() {
@@ -125,8 +125,8 @@ class UpdatePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
         then: 'data leaves have expected values'
             assert 100 == countDataNodes('/openroadm-devices/openroadm-device[@status="fail"]')
-        and: 'update completes within expected time and a margin of 50%'
-            recordAndAssertResourceUsage('Update leaves for 100 data nodes', 0.14, resourceMeter.getTotalTimeInSeconds(), 120, resourceMeter.getTotalMemoryUsageInMB())
+        and: 'update completes within expected time and a margin of 100%'
+            recordAndAssertResourceUsage('Update leaves for 100 data nodes', 0.22, resourceMeter.getTotalTimeInSeconds(), 120, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Clean up for CPS Update API.'() {

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy
index 21a9f19..642f3f3 100644 (file)
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy
@@ -39,18 +39,16 @@ class WritePerfTest extends CpsPerfTestBase {
             resourceMeter.start()
             cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, jsonData, OffsetDateTime.now())
             resourceMeter.stop()
-        then: 'the operation takes less than #expectedDuration with a margin of 50%'
-            recordAndAssertResourceUsage("Writing ${totalNodes} devices",
-                    expectedDuration, resourceMeter.getTotalTimeInSeconds(),
-                    memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
+        then: 'the operation takes less than #expectedDuration with a margin of 100%'
+            recordAndAssertResourceUsage("Writing ${totalNodes} devices", expectedDuration, resourceMeter.getTotalTimeInSeconds(), memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
         cleanup:
             cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR)
         where:
             totalNodes || expectedDuration | memoryLimit
             50         || 1.45             | 100
             100        || 2.9              | 200
-            200        || 5.6              | 400
-            400        || 11.4             | 500
+            200        || 6.2              | 400
+            400        || 13.0             | 500
     }
 
     def 'Writing bookstore data has exponential time.'() {
@@ -64,7 +62,7 @@ class WritePerfTest extends CpsPerfTestBase {
             resourceMeter.start()
             cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, '/bookstore/categories[@code=1]', booksData, OffsetDateTime.now())
             resourceMeter.stop()
-        then: 'the operation takes less than #expectedDuration with a margin of 50%'
+        then: 'the operation takes less than #expectedDuration with a margin of 100%'
             recordAndAssertResourceUsage("Writing ${totalBooks} books",
                     expectedDuration, resourceMeter.totalTimeInSeconds,
                     memoryLimit, resourceMeter.totalMemoryUsageInMB)
@@ -72,8 +70,8 @@ class WritePerfTest extends CpsPerfTestBase {
             cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR)
         where:
             totalBooks || expectedDuration | memoryLimit
-            800        || 0.26             | 50
-            1600       || 0.75             | 100
+            800        || 0.31             | 50
+            1600       || 0.8              | 100
             3200       || 2.2              | 150
             6400       || 6.9              | 200
     }
@@ -92,18 +90,16 @@ class WritePerfTest extends CpsPerfTestBase {
             resourceMeter.start()
             cpsDataService.saveListElements(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR, '/openroadm-devices', jsonListData, OffsetDateTime.now(), ContentType.JSON)
             resourceMeter.stop()
-        then: 'the operation takes less than #expectedDuration with a margin of 50%'
-            recordAndAssertResourceUsage("Saving list of ${totalNodes} devices",
-                    expectedDuration, resourceMeter.totalTimeInSeconds,
-                    memoryLimit, resourceMeter.totalMemoryUsageInMB)
+        then: 'the operation takes less than #expectedDuration with a margin of 100%'
+            recordAndAssertResourceUsage("Saving list of ${totalNodes} devices", expectedDuration, resourceMeter.totalTimeInSeconds, memoryLimit, resourceMeter.totalMemoryUsageInMB)
         cleanup:
             cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, WRITE_TEST_ANCHOR)
         where:
             totalNodes || expectedDuration | memoryLimit
-            50         || 1.35             | 100
-            100        || 2.7              | 200
-            200        || 5.4              | 400
-            400        || 10.8             | 500
+            50         || 1.5              | 100
+            100        || 3.0              | 200
+            200        || 7.1              | 400
+            400        || 12.5             | 500
     }
 
 }

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy
index b4456e3..a7a1760 100644 (file)
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy
@@ -97,7 +97,7 @@ class CmDataSubscriptionsPerfTest extends NcmpPerfTestBase {
             def resultAfter = objectUnderTest.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_ANCHOR, cpsPath, INCLUDE_ALL_DESCENDANTS)
             assert resultAfter.collect {it.leaves.subscribers.size()}.sum() == totalNumberOfEntries * (1 + numberOfCmDataSubscribers)
         and: 'update matching subscription within 15 seconds'
-            recordAndAssertResourceUsage("Update matching subscription", 6.2, durationInSeconds, 1000, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage("Update matching subscription", 10.5, durationInSeconds, 1000, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Worst case new subscription (200x10 new entries).'() {

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy
index 5f04519..3216cc1 100644 (file)
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy
@@ -80,7 +80,7 @@ class CmHandleQueryPerfTest extends NcmpPerfTestBase {
         then: '100 data node have been found'
             assert count == 100
         and: 'average performance is as expected'
-            recordAndAssertResourceUsage('Look up CM-handle by id', 0.25, resourceMeter.totalTimeInSeconds, 15, resourceMeter.totalMemoryUsageInMB)
+            recordAndAssertResourceUsage('Look up CM-handle by id', 0.58, resourceMeter.totalTimeInSeconds, 15, resourceMeter.totalMemoryUsageInMB)
     }
 
     def 'CM-handle is looked up by alternate id.'() {
@@ -95,7 +95,7 @@ class CmHandleQueryPerfTest extends NcmpPerfTestBase {
         then: 'all alternate ids are resolved correctly'
             assert count == 100
         and: 'average performance is as expected'
-            recordAndAssertResourceUsage('Look up CM-handle by alternate-id', 1.75, resourceMeter.totalTimeInSeconds, 15, resourceMeter.totalMemoryUsageInMB)
+            recordAndAssertResourceUsage('Look up CM-handle by alternate-id', 1.4, resourceMeter.totalTimeInSeconds, 15, resourceMeter.totalMemoryUsageInMB)
     }
 
     def 'A batch of CM-handles is looked up by alternate id.'() {
@@ -125,7 +125,7 @@ class CmHandleQueryPerfTest extends NcmpPerfTestBase {
             assert count == TOTAL_CM_HANDLES * 100
         then: 'average performance is as expected'
             def averageResponseTime = resourceMeter.totalTimeInSeconds / 100
-            recordAndAssertResourceUsage('Look up CM-handles by module-set-tag', 0.26, averageResponseTime, 500, resourceMeter.totalMemoryUsageInMB)
+            recordAndAssertResourceUsage('Look up CM-handles by module-set-tag', 0.25, averageResponseTime, 500, resourceMeter.totalMemoryUsageInMB)
     }
 
 }