Adjust performance test timings

New limits for time and memory are based on analyzing 40 previous
performance job runs. Test limits are chosen so that individual
test cases would pass 95% of the the time.

Issue-ID: CPS-1884
Signed-off-by: danielhanrahan <daniel.hanrahan@est.tech>
Change-Id: Iadca27eb1f9482016759070e5afdc70c53f9d043

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy
index 8cdbdc5..d8012ec 100644 (file)
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy
@@ -67,7 +67,7 @@ class CpsPerfTestBase extends PerfTestBase {
         addAnchorsWithData(OPENROADM_ANCHORS, CPS_PERFORMANCE_TEST_DATASPACE, LARGE_SCHEMA_SET, 'openroadm', data)
         resourceMeter.stop()
         def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
-        recordAndAssertResourceUsage('Creating openroadm anchors with large data tree', 200, durationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB())
+        recordAndAssertResourceUsage('Creating openroadm anchors with large data tree', 200, durationInSeconds, 600, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def generateOpenRoadData(numberOfNodes) {
@@ -87,7 +87,7 @@ class CpsPerfTestBase extends PerfTestBase {
         then: 'memory used is within #peakMemoryUsage'
             assert resourceMeter.getTotalMemoryUsageInMB() <= 30
         and: 'all data is read within expected time'
-            recordAndAssertResourceUsage("Warming database", 200, durationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage("Warming database", 200, durationInSeconds, 600, resourceMeter.getTotalMemoryUsageInMB())
     }
 
 }

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy
index b0a105c..a96d7f6 100644 (file)
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy
@@ -55,7 +55,7 @@ abstract class PerfTestBase extends CpsIntegrationSpecBase {
     abstract def createInitialData()
 
     def recordAndAssertResourceUsage(String shortTitle, double thresholdInSec, double recordedTimeInSec, memoryLimit, memoryUsageInMB) {
-        def pass = recordedTimeInSec <= thresholdInSec
+        def pass = recordedTimeInSec <= thresholdInSec && memoryUsageInMB <= memoryLimit
         if (shortTitle.length() > 40) {
             shortTitle = shortTitle.substring(0, 40)
         }

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy
index 6a3d443..ce66cb0 100644 (file)
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy
@@ -47,7 +47,7 @@ class CpsDataServiceLimitsPerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'the operation completes within 25 seconds'
-            recordAndAssertResourceUsage("Creating 33,000 books", 25, durationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage("Creating 33,000 books", 25, durationInSeconds, 150, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Get data nodes from multiple xpaths 32K (2^15) limit exceeded.'() {
@@ -88,7 +88,7 @@ class CpsDataServiceLimitsPerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'test data is deleted in 1 second'
-            recordAndAssertResourceUsage("Deleting test data", 1, durationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage("Deleting test data", 1, durationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def countDataNodes() {

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy
index 0dcd995..818c300 100644 (file)
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy
@@ -40,7 +40,7 @@ class DeletePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def setupDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'setup duration is within expected time and memory used is within limit'
-            recordAndAssertResourceUsage('Delete test setup', 200, setupDurationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage('Delete test setup', 200, setupDurationInSeconds, 800, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Delete 100 container nodes'() {
@@ -56,7 +56,7 @@ class DeletePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'delete duration is within expected time and memory used is within limit'
-            recordAndAssertResourceUsage('Delete 100 containers', 2, deleteDurationInSeconds, 30, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage('Delete 100 containers', 2.5, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Batch delete 100 container nodes'() {
@@ -70,7 +70,7 @@ class DeletePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'delete duration is within expected time and memory used is within limit'
-            recordAndAssertResourceUsage('Batch delete 100 containers', 0.5, deleteDurationInSeconds, 5, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage('Batch delete 100 containers', 0.6, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Delete 100 list elements'() {
@@ -86,7 +86,7 @@ class DeletePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'delete duration is within expected time and memory used is within limit'
-            recordAndAssertResourceUsage('Delete 100 lists elements', 2, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage('Delete 100 lists elements', 2.5, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Batch delete 100 list elements'() {
@@ -100,7 +100,7 @@ class DeletePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'delete duration is within expected time and memory used is within limit'
-            recordAndAssertResourceUsage('Batch delete 100 lists elements', 0.5, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage('Batch delete 100 lists elements', 0.6, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Delete 100 whole lists'() {
@@ -116,7 +116,7 @@ class DeletePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'delete duration is within expected time and memory used is within limit'
-            recordAndAssertResourceUsage('Delete 100 whole lists', 5, deleteDurationInSeconds, 30, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage('Delete 100 whole lists', 6, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Batch delete 100 whole lists'() {
@@ -130,7 +130,7 @@ class DeletePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'delete duration is within expected time and memory used is within limit'
-            recordAndAssertResourceUsage('Batch delete 100 whole lists', 4, deleteDurationInSeconds, 5, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage('Batch delete 100 whole lists', 5, deleteDurationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Delete 1 large data node'() {
@@ -140,7 +140,7 @@ class DeletePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'delete duration is within expected time and memory used is within limit'
-            recordAndAssertResourceUsage('Delete one large node', 2, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage('Delete one large node', 2, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Delete root node with many descendants'() {
@@ -150,7 +150,7 @@ class DeletePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'delete duration is within expected time and memory used is within limit'
-            recordAndAssertResourceUsage('Delete root node', 2, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage('Delete root node', 2, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Delete data nodes for an anchor'() {
@@ -160,7 +160,7 @@ class DeletePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'delete duration is within expected time and memory used is within limit'
-            recordAndAssertResourceUsage('Delete data nodes for anchor', 2, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage('Delete data nodes for anchor', 2, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Batch delete 100 non-existing nodes'() {
@@ -174,7 +174,7 @@ class DeletePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'delete duration is within expected time and memory used is within limit'
-            recordAndAssertResourceUsage('Batch delete 100 non-existing', 7, deleteDurationInSeconds, 5, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage('Batch delete 100 non-existing', 7, deleteDurationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Clean up test data'() {
@@ -186,7 +186,7 @@ class DeletePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'delete duration is within expected time and memory used is within limit'
-            recordAndAssertResourceUsage('Delete test cleanup', 10, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage('Delete test cleanup', 10, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB())
     }
 
 }

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy
index 95cf260..8a228a3 100644 (file)
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy
@@ -44,9 +44,9 @@ class GetPerfTest extends CpsPerfTestBase {
             recordAndAssertResourceUsage("Read datatrees with ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
         where: 'the following parameters are used'
             scenario             | fetchDescendantsOption  || durationLimit | memoryLimit  | expectedNumberOfDataNodes
-            'no descendants'     | OMIT_DESCENDANTS        || 0.01          | 5            | 1
-            'direct descendants' | DIRECT_CHILDREN_ONLY    || 0.05          | 10           | 1 + OPENROADM_DEVICES_PER_ANCHOR
-            'all descendants'    | INCLUDE_ALL_DESCENDANTS || 2             | 200          | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
+            'no descendants'     | OMIT_DESCENDANTS        || 0.02          | 1            | 1
+            'direct descendants' | DIRECT_CHILDREN_ONLY    || 0.06          | 5            | 1 + OPENROADM_DEVICES_PER_ANCHOR
+            'all descendants'    | INCLUDE_ALL_DESCENDANTS || 2.5           | 250          | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
     }
 
     def 'Read data trees for multiple xpaths'() {
@@ -60,7 +60,7 @@ class GetPerfTest extends CpsPerfTestBase {
         then: 'requested nodes and their descendants are returned'
             assert countDataNodesInTree(result) == OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
         and: 'all data is read within expected time and memory used is within limit'
-            recordAndAssertResourceUsage("Read datatrees for multiple xpaths", 3 , durationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage("Read datatrees for multiple xpaths", 4 , durationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Read for multiple xpaths to non-existing datanodes'() {
@@ -74,7 +74,7 @@ class GetPerfTest extends CpsPerfTestBase {
         then: 'no data is returned'
             assert result.isEmpty()
         and: 'the operation completes within within expected time'
-            recordAndAssertResourceUsage("Read non-existing xpaths", 0.01, durationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage("Read non-existing xpaths", 0.02, durationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Read complete data trees using #scenario.'() {
@@ -88,9 +88,9 @@ class GetPerfTest extends CpsPerfTestBase {
             recordAndAssertResourceUsage("Read datatrees using ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
         where: 'the following xpaths are used'
             scenario                | xpath                                  || durationLimit  | memoryLimit  | expectedNumberOfDataNodes
-            'openroadm root'        | '/'                                    || 2              | 200          | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
-            'openroadm top element' | '/openroadm-devices'                   || 2              | 200          | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
-            'openroadm whole list'  | '/openroadm-devices/openroadm-device'  || 3              | 200          | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
+            'openroadm root'        | '/'                                    || 2              | 250          | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
+            'openroadm top element' | '/openroadm-devices'                   || 2              | 250          | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
+            'openroadm whole list'  | '/openroadm-devices/openroadm-device'  || 3              | 250          | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
     }
 
 }

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy
index 5cf4955..0ae018d 100644 (file)
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy
@@ -45,10 +45,11 @@ class QueryPerfTest extends CpsPerfTestBase {
             recordAndAssertResourceUsage("Query 1 anchor ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
         where: 'the following parameters are used'
             scenario                     | cpsPath                                                             || durationLimit  | memoryLimit  | expectedNumberOfDataNodes
-            'top element'                | '/openroadm-devices'                                                || 2              | 300          | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1
-            'leaf condition'             | '//openroadm-device[@ne-state="inservice"]'                         || 3              | 200          | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
-            'ancestors'                  | '//openroadm-device/ancestor::openroadm-devices'                    || 2              | 200          | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1
-            'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 2              | 300          | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1
+            'top element'                | '/openroadm-devices'                                                || 2.5            | 400          | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1
+            'leaf condition'             | '//openroadm-device[@ne-state="inservice"]'                         || 2.5            | 400          | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
+            'ancestors'                  | '//openroadm-device/ancestor::openroadm-devices'                    || 2.5            | 400          | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1
+            'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 2.5            | 400          | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1
+            'non-existing data'          | '/path/to/non-existing/node[@id="1"]'                               || 0.1            | 1            | 0
     }
 
     def 'Query complete data trees across all anchors with #scenario.'() {
@@ -63,11 +64,10 @@ class QueryPerfTest extends CpsPerfTestBase {
             recordAndAssertResourceUsage("Query across anchors ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
         where: 'the following parameters are used'
             scenario                     | cpspath                                                             || durationLimit  | memoryLimit   | expectedNumberOfDataNodes
-            'top element'                | '/openroadm-devices'                                                || 6              | 600           | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1)
-            'leaf condition'             | '//openroadm-device[@ne-state="inservice"]'                         || 6              | 600           | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE)
-            'ancestors'                  | '//openroadm-device/ancestor::openroadm-devices'                    || 6              | 800           | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1)
-            'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 6              | 600           | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1)
-            'non-existing data'          | '/path/to/non-existing/node[@id="1"]'                               || 0.1            | 3             | 0
+            'top element'                | '/openroadm-devices'                                                || 7              | 600           | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1)
+            'leaf condition'             | '//openroadm-device[@ne-state="inservice"]'                         || 7              | 600           | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE)
+            'ancestors'                  | '//openroadm-device/ancestor::openroadm-devices'                    || 7              | 600           | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1)
+            'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 7              | 600           | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1)
     }
 
     def 'Query with leaf condition and #scenario.'() {
@@ -82,9 +82,9 @@ class QueryPerfTest extends CpsPerfTestBase {
             recordAndAssertResourceUsage("Query with ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
         where: 'the following parameters are used'
             scenario             | fetchDescendantsOption  || durationLimit  | memoryLimit   | expectedNumberOfDataNodes
-            'no descendants'     | OMIT_DESCENDANTS        || 0.1            | 30            | OPENROADM_DEVICES_PER_ANCHOR
-            'direct descendants' | DIRECT_CHILDREN_ONLY    || 0.15           | 30            | OPENROADM_DEVICES_PER_ANCHOR * 2
-            'all descendants'    | INCLUDE_ALL_DESCENDANTS || 2              | 200           | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
+            'no descendants'     | OMIT_DESCENDANTS        || 0.1            | 6             | OPENROADM_DEVICES_PER_ANCHOR
+            'direct descendants' | DIRECT_CHILDREN_ONLY    || 0.2            | 12            | OPENROADM_DEVICES_PER_ANCHOR * 2
+            'all descendants'    | INCLUDE_ALL_DESCENDANTS || 2.5            | 200           | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
     }
 
     def 'Query ancestors with #scenario.'() {
@@ -99,9 +99,9 @@ class QueryPerfTest extends CpsPerfTestBase {
             recordAndAssertResourceUsage("Query ancestors with ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
         where: 'the following parameters are used'
             scenario             | fetchDescendantsOption  || durationLimit  | memoryLimit | expectedNumberOfDataNodes
-            'no descendants'     | OMIT_DESCENDANTS        || 0.1            | 20          | 1
-            'direct descendants' | DIRECT_CHILDREN_ONLY    || 0.1            | 20          | 1 + OPENROADM_DEVICES_PER_ANCHOR
-            'all descendants'    | INCLUDE_ALL_DESCENDANTS || 2              | 200         | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
+            'no descendants'     | OMIT_DESCENDANTS        || 0.1            | 3           | 1
+            'direct descendants' | DIRECT_CHILDREN_ONLY    || 0.2            | 8           | 1 + OPENROADM_DEVICES_PER_ANCHOR
+            'all descendants'    | INCLUDE_ALL_DESCENDANTS || 2.5            | 400         | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE
     }
 
 }

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy
index 151492d..8118010 100644 (file)
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy
@@ -41,7 +41,7 @@ class UpdatePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def updateDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'update completes within expected time and memory used is within limit'
-            recordAndAssertResourceUsage('Update 1 data node', 0.6, updateDurationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage('Update 1 data node', 0.6, updateDurationInSeconds, 100, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Batch update 100 data nodes with descendants'() {
@@ -57,7 +57,7 @@ class UpdatePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def updateDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'update completes within expected time and memory used is within limit'
-            recordAndAssertResourceUsage('Update 100 data nodes', 30, updateDurationInSeconds, 800, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage('Update 100 data nodes', 40, updateDurationInSeconds, 800, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Update leaves for 1 data node (twice)'() {
@@ -71,7 +71,7 @@ class UpdatePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def updateDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'update completes within expected time and memory used is within limit'
-            recordAndAssertResourceUsage('Update leaves for 1 data node', 0.5, updateDurationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage('Update leaves for 1 data node', 0.7, updateDurationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Batch update leaves for 100 data nodes (twice)'() {
@@ -85,7 +85,7 @@ class UpdatePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def updateDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'update completes within expected time and memory used is within limit'
-            recordAndAssertResourceUsage('Batch update leaves for 100 data nodes', 1, updateDurationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage('Batch update leaves for 100 data nodes', 1, updateDurationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB())
     }
 
 }

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy
index 177cd9f..0c7107a 100644 (file)
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy
@@ -36,19 +36,16 @@ class WritePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'the operation takes less than #expectedDuration and memory used is within limit'
-            recordAndAssertResourceUsage("Writing ${totalNodes} devices", expectedDurationInSeconds, durationInSeconds, 400, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage("Writing ${totalNodes} devices", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
         cleanup:
             cpsDataService.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor', OffsetDateTime.now())
             cpsAdminService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor')
         where:
-            totalNodes || expectedDurationInSeconds
-            50         ||   3
-            100        ||   5
-            200        ||  10
-            400        ||  20
-//          800        ||  40
-//          1600       ||  80
-//          3200       || 160
+            totalNodes || expectedDuration | memoryLimit
+            50         || 4                | 100
+            100        || 7                | 200
+            200        || 14               | 400
+            400        || 28               | 500
     }
 
     def 'Writing bookstore data has exponential time.'() {
@@ -64,20 +61,16 @@ class WritePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'the operation takes less than #expectedDuration and memory used is within limit'
-            recordAndAssertResourceUsage("Writing ${totalBooks} books", expectedDuration, durationInSeconds, 400, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage("Writing ${totalBooks} books", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB())
         cleanup:
             cpsDataService.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor', OffsetDateTime.now())
             cpsAdminService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor')
         where:
-            totalBooks || expectedDuration
-            400        || 0.2
-            800        || 0.5
-            1600       || 1
-            3200       || 3
-            6400       || 10
-//          12800      || 30
-//          25600      || 120
-//          51200      || 600
+            totalBooks || expectedDuration | memoryLimit
+            800        || 1                | 50
+            1600       || 2                | 100
+            3200       || 6                | 150
+            6400       || 18               | 200
     }
 
 }

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy
index 896217a..579394b 100644 (file)
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy
@@ -52,7 +52,7 @@ class CmDataSubscriptionsPerfTest extends NcmpPerfTestBase {
             matches.size() == numberOfFiltersPerCmHandle * numberOfCmHandlesPerCmDataSubscription
         and: 'query all subscribers within 1 second'
             def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
-            recordAndAssertResourceUsage("Query all subscribers", 1, durationInSeconds, 400, resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage("Query all subscribers", 1.2, durationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Worst case subscription update (200x10 matching entries).'() {
@@ -94,8 +94,8 @@ class CmDataSubscriptionsPerfTest extends NcmpPerfTestBase {
         then: 'a subscriber has been added to each filter entry'
             def resultAfter = objectUnderTest.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_ANCHOR, cpsPath, INCLUDE_ALL_DESCENDANTS)
             assert resultAfter.collect {it.leaves.subscribers.size()}.sum() == totalNumberOfEntries * (1 + numberOfCmDataSubscribers)
-        and: 'update matching subscription within 8 seconds'
-            recordAndAssertResourceUsage("Update matching subscription", 8, durationInSeconds, 400, resourceMeter.getTotalMemoryUsageInMB())
+        and: 'update matching subscription within 15 seconds'
+            recordAndAssertResourceUsage("Update matching subscription", 15, durationInSeconds, 1000, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def 'Worst case new subscription (200x10 new entries).'() {
@@ -109,7 +109,7 @@ class CmDataSubscriptionsPerfTest extends NcmpPerfTestBase {
             resourceMeter.stop()
             def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'insert new subscription with 1 second'
-            recordAndAssertResourceUsage("Insert new subscription", 1, durationInSeconds, 400,resourceMeter.getTotalMemoryUsageInMB())
+            recordAndAssertResourceUsage("Insert new subscription", 2, durationInSeconds, 100, resourceMeter.getTotalMemoryUsageInMB())
     }
 
     def querySubscriptionsByIteration(Collection<DataNode> allSubscriptionsAsDataNodes, targetSubscriptionSequenceNumber) {

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy
index becd7e3..a5a6acb 100644 (file)
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy
@@ -45,8 +45,8 @@ class CmHandleQueryPerfTest extends NcmpPerfTestBase {
             def result = cpsDataService.getDataNodesForMultipleXpaths(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR, xpaths, INCLUDE_ALL_DESCENDANTS)
             resourceMeter.stop()
             def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
-        then: 'the required operations are performed within 1200 ms'
-            recordAndAssertResourceUsage("CpsPath Registry attributes Query", 0.25, durationInSeconds, 150, resourceMeter.getTotalMemoryUsageInMB())
+        then: 'the required operations are performed within required time'
+            recordAndAssertResourceUsage("CpsPath Registry attributes Query", 0.4, durationInSeconds, 50, resourceMeter.getTotalMemoryUsageInMB())
         and: 'all but 1 (other node) are returned'
             result.size() == 999
         and: 'the tree contains all the expected descendants too'

diff --git a/integration-test/src/test/java/org/onap/cps/integration/ResourceMeter.java b/integration-test/src/test/java/org/onap/cps/integration/ResourceMeter.java
index 1e42001..c7d96c4 100644 (file)
--- a/integration-test/src/test/java/org/onap/cps/integration/ResourceMeter.java
+++ b/integration-test/src/test/java/org/onap/cps/integration/ResourceMeter.java
@@ -48,8 +48,8 @@ public class ResourceMeter {
     }
 
     /**
-     * Get the total time in milliseconds.
-     * @return total time in milliseconds
+     * Get the total time in seconds.
+     * @return total time in seconds
      */
     public double getTotalTimeInSeconds() {
         return stopWatch.getTotalTimeSeconds();