Make Groovy Performance Test Stable

- Overload assert method to apply custom margin
- Increase margin to 3x (+200%) for tests
  Chosen based on last 2 failures and graph data

Issue-ID: CPS-2799
Change-Id: I5767d529ed86c6a57f2213639834fec3fb0be0b8
Signed-off-by: ToineSiebelink <toine.siebelink@est.tech>

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy
index c89670f..145aeb0 100644 (file)
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy
@@ -56,11 +56,7 @@ abstract class PerfTestBase extends CpsIntegrationSpecBase {
 
     abstract def createInitialData()
 
-    def recordAndAssertResourceUsage(String title, double expectedAverageTimeInSec, double recordedTimeInSec, double memoryUsageInMB, boolean referenceGraph) {
-        def timeLimitFactor = DEFAULT_TIME_LIMIT_FACTOR
-        if (expectedAverageTimeInSec <= VERY_FAST_TEST_THRESHOLD) {
-            timeLimitFactor = DEFAULT_TIME_LIMIT_FACTOR_FOR_VERY_FAST_TEST
-        }
+    def recordAndAssertResourceUsage(String title, double expectedAverageTimeInSec, double recordedTimeInSec, double memoryUsageInMB, boolean referenceGraph, double timeLimitFactor) {
         def testPassed = recordedTimeInSec <= timeLimitFactor * expectedAverageTimeInSec
         addRecord(title, expectedAverageTimeInSec, recordedTimeInSec, memoryUsageInMB, testPassed)
         if (referenceGraph) {
@@ -71,6 +67,15 @@ abstract class PerfTestBase extends CpsIntegrationSpecBase {
         return true
     }
 
+    def recordAndAssertResourceUsage(String title, double expectedAverageTimeInSec, double recordedTimeInSec, double memoryUsageInMB, boolean referenceGraph) {
+        def timeLimitFactor = DEFAULT_TIME_LIMIT_FACTOR
+        if (expectedAverageTimeInSec <= VERY_FAST_TEST_THRESHOLD) {
+            timeLimitFactor = DEFAULT_TIME_LIMIT_FACTOR_FOR_VERY_FAST_TEST
+        }
+        recordAndAssertResourceUsage(title, expectedAverageTimeInSec, recordedTimeInSec, memoryUsageInMB, referenceGraph, timeLimitFactor)
+
+    }
+
     def recordAndAssertResourceUsage(String title, double expectedAverageTimeInSec, double recordedTimeInSec, double memoryUsageInMB) {
         recordAndAssertResourceUsage(title, expectedAverageTimeInSec, recordedTimeInSec, memoryUsageInMB, false)
     }

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy
index 8d4d02c..f754582 100644 (file)
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy
@@ -74,7 +74,7 @@ class CpsDataServiceLimitsPerfTest extends CpsPerfTestBase {
     def 'Delete data nodes from multiple anchors 32K (2^15) limit exceeded.'() {
         given: '33,000 anchor names'
             def anchorNames = (1..33_000).collect { "size-of-this-name-does-not-matter-for-limit-" + it }
-        when: 'a single operation is executed to delete all datanodes in given anchors'
+        when: 'a single operation is executed to delete all data nodes in given anchors'
             objectUnderTest.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, anchorNames, OffsetDateTime.now())
         then: 'a database exception is not thrown'
             noExceptionThrown()
@@ -87,8 +87,8 @@ class CpsDataServiceLimitsPerfTest extends CpsPerfTestBase {
             cpsAnchorService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, 'limitsAnchor')
             resourceMeter.stop()
             def durationInSeconds = resourceMeter.getTotalTimeInSeconds()
-        then: 'test data is deleted in 1 second'
-            recordAndAssertResourceUsage('CPS:Deleting test data', 0.068, durationInSeconds, resourceMeter.getTotalMemoryUsageInMB())
+        then: 'test data is deleted in 1 second (margin 3x)'
+            recordAndAssertResourceUsage('CPS:Deleting test data', 0.068, durationInSeconds, resourceMeter.getTotalMemoryUsageInMB(), false, 3)
     }
 
     def countDataNodes() {

diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy
index 043507f..cf8ea11 100644 (file)
--- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy
+++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy
@@ -116,8 +116,8 @@ class DeletePerfTest extends CpsPerfTestBase {
             }
             resourceMeter.stop()
             def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
-        then: 'delete duration is below accepted margin (4x) of the expected average'
-            recordAndAssertResourceUsage('CPS:Delete 100 whole lists', 3.2, deleteDurationInSeconds, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH)
+        then: 'delete duration is below accepted margin (3x) of the expected average'
+            recordAndAssertResourceUsage('CPS:Delete 100 whole lists', 3.2, deleteDurationInSeconds, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH, 3)
     }
 
     def 'Batch delete 100 whole lists'() {
@@ -129,7 +129,7 @@ class DeletePerfTest extends CpsPerfTestBase {
             resourceMeter.stop()
             def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds()
         then: 'delete duration is below accepted margin (3x) of the expected average'
-            recordAndAssertResourceUsage('CPS:Batch delete 100 whole lists', 2.4, deleteDurationInSeconds, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH)
+            recordAndAssertResourceUsage('CPS:Batch delete 100 whole lists', 2.2, deleteDurationInSeconds, resourceMeter.getTotalMemoryUsageInMB(), REFERENCE_GRAPH, 3)
     }
 
     def 'Delete 1 large data node'() {