From: Toine Siebelink Date: Tue, 17 Oct 2023 13:17:08 +0000 (+0000) Subject: Merge "Add memory usage to integration tests [UPDATED]" X-Git-Tag: 3.3.9~14 X-Git-Url: https://gerrit.onap.org/r/gitweb?a=commitdiff_plain;h=c29a5fe32db7efb0fdefa3a3d020110aab9731b4;hp=fc7a5d30953cfedd7e2ea2f969adab03716de6df;p=cps.git Merge "Add memory usage to integration tests [UPDATED]" --- diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy index 5fd27476c..e7a847e13 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy @@ -20,6 +20,7 @@ package org.onap.cps.integration.performance.base +import org.onap.cps.integration.ResourceMeter import org.onap.cps.rest.utils.MultipartFileUtil import org.onap.cps.spi.FetchDescendantsOption import org.springframework.web.multipart.MultipartFile @@ -33,6 +34,8 @@ class CpsPerfTestBase extends PerfTestBase { static final def OPENROADM_DEVICES_PER_ANCHOR = 1000 static final def OPENROADM_DATANODES_PER_DEVICE = 86 + ResourceMeter resourceMeter = new ResourceMeter() + def printTitle() { println('## C P S P E R F O R M A N C E T E S T R E S U L T S ##') } @@ -62,11 +65,11 @@ class CpsPerfTestBase extends PerfTestBase { def addOpenRoadData() { def data = generateOpenRoadData(OPENROADM_DEVICES_PER_ANCHOR) - stopWatch.start() + resourceMeter.start() addAnchorsWithData(OPENROADM_ANCHORS, CPS_PERFORMANCE_TEST_DATASPACE, LARGE_SCHEMA_SET, 'openroadm', data) - stopWatch.stop() - def durationInMillis = stopWatch.getTotalTimeMillis() - recordAndAssertPerformance('Creating openroadm anchors with large data tree', TimeUnit.SECONDS.toMillis(200), durationInMillis) + resourceMeter.stop() + def durationInMillis = resourceMeter.getTotalTimeMillis() + recordAndAssertResourceUsage('Creating openroadm anchors with large data tree', TimeUnit.SECONDS.toMillis(200), durationInMillis, 200, resourceMeter.getTotalMemoryUsageInMB()) } def generateOpenRoadData(numberOfNodes) { @@ -78,13 +81,15 @@ class CpsPerfTestBase extends PerfTestBase { def 'Warm the database'() { when: 'dummy get data nodes runs so that populating the DB does not get included in other test timings' - stopWatch.start() + resourceMeter.start() def result = cpsDataService.getDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'openroadm1', '/', FetchDescendantsOption.OMIT_DESCENDANTS) assert countDataNodesInTree(result) == 1 - stopWatch.stop() - def durationInMillis = stopWatch.getTotalTimeMillis() - then: 'all data is read within expected time' - recordAndAssertPerformance("Warming database", TimeUnit.SECONDS.toMillis(200), durationInMillis) + resourceMeter.stop() + def durationInMillis = resourceMeter.getTotalTimeMillis() + then: 'memory used is within #peakMemoryUsage' + assert resourceMeter.getTotalMemoryUsageInMB() <= 30 + and: 'all data is read within expected time' + recordAndAssertResourceUsage("Warming database", TimeUnit.SECONDS.toMillis(200), durationInMillis, 200, resourceMeter.getTotalMemoryUsageInMB()) } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/NcmpPerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/NcmpPerfTestBase.groovy index f5d7c5e15..7bacf1d6e 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/NcmpPerfTestBase.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/NcmpPerfTestBase.groovy @@ -21,6 +21,7 @@ package org.onap.cps.integration.performance.base import java.time.OffsetDateTime +import org.onap.cps.integration.ResourceMeter class NcmpPerfTestBase extends PerfTestBase { @@ -36,6 +37,8 @@ class NcmpPerfTestBase extends PerfTestBase { def numberOfFiltersPerCmHandle = 10 def numberOfCmHandlesPerCmDataSubscription = 200 + ResourceMeter resourceMeter = new ResourceMeter() + // SHORT versions for easier debugging // def subscriberIdPrefix = 'sub' // def xpathPrefix = 'f' diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy index b6ceb91b5..8e5fe0619 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy @@ -28,8 +28,6 @@ abstract class PerfTestBase extends CpsIntegrationSpecBase { static def LARGE_SCHEMA_SET = 'largeSchemaSet' static def PERFORMANCE_RECORD = [] - def stopWatch = new StopWatch() - def cleanupSpec() { println('#############################################################################') printTitle() @@ -56,15 +54,16 @@ abstract class PerfTestBase extends CpsIntegrationSpecBase { abstract def createInitialData() - def recordAndAssertPerformance(String shortTitle, thresholdInMs, recordedTimeInMs) { + def recordAndAssertResourceUsage(String shortTitle, thresholdInMs, recordedTimeInMs, memoryLimit, memoryUsageInMB) { def pass = recordedTimeInMs <= thresholdInMs if (shortTitle.length() > 40) { shortTitle = shortTitle.substring(0, 40) } - def record = String.format('%2d.%-40s limit%,8d took %,8d ms ', PERFORMANCE_RECORD.size() + 1, shortTitle, thresholdInMs, recordedTimeInMs) + def record = String.format('%2d.%-40s limit%,8d took %,8d ms %,8.2f MB used ', PERFORMANCE_RECORD.size() + 1, shortTitle, thresholdInMs, recordedTimeInMs, memoryUsageInMB) record += pass ? 'PASS' : 'FAIL' PERFORMANCE_RECORD.add(record) assert recordedTimeInMs <= thresholdInMs + assert memoryUsageInMB <= memoryLimit return true } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy index 659c9f579..1aea1235e 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy @@ -41,15 +41,15 @@ class CpsDataServiceLimitsPerfTest extends CpsPerfTestBase { def parentNodeData = '{"bookstore": { "categories": [{ "code": 1, "name": "Test", "books" : [] }] }}' cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, 'limitsAnchor', parentNodeData, OffsetDateTime.now()) when: '33,000 books are added' - stopWatch.start() + resourceMeter.start() for (int i = 1; i <= 33_000; i+=100) { def booksData = '{"books":[' + (i..> filterEntriesPerPath = [:] matches.each { dataNode, subscribersAsArray -> def updatedSubscribers = createLeafList('subscribers', 1 + numberOfCmDataSubscribers, subscriberIdPrefix) @@ -89,13 +89,13 @@ class CmDataSubscriptionsPerfTest extends NcmpPerfTestBase { cpsDataService.updateNodeLeaves(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_ANCHOR, parentPath, json, now) } - stopWatch.stop() - def durationInMillis = stopWatch.getTotalTimeMillis() + resourceMeter.stop() + def durationInMillis = resourceMeter.getTotalTimeMillis() then: 'a subscriber has been added to each filter entry' def resultAfter = objectUnderTest.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_ANCHOR, cpsPath, INCLUDE_ALL_DESCENDANTS) assert resultAfter.collect {it.leaves.subscribers.size()}.sum() == totalNumberOfEntries * (1 + numberOfCmDataSubscribers) and: 'update matching subscription within 8 seconds' - recordAndAssertPerformance("Update matching subscription", 8_000, durationInMillis) + recordAndAssertResourceUsage("Update matching subscription", 8_000, durationInMillis, 400, resourceMeter.getTotalMemoryUsageInMB()) } def 'Worst case new subscription (200x10 new entries).'() { @@ -104,12 +104,12 @@ class CmDataSubscriptionsPerfTest extends NcmpPerfTestBase { def filters = '"filters":' + createJsonArray('filter',numberOfFiltersPerCmHandle,'xpath','other_' + xpathPrefix,subscribers) def cmHandles = createJsonArray('cm-handle',numberOfCmHandlesPerCmDataSubscription,'id','other' + cmHandlePrefix, filters) when: 'Insert a new subscription' - stopWatch.start() + resourceMeter.start() cpsDataService.saveData(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_ANCHOR, xPathForDataStore1CmHandles, cmHandles, now) - stopWatch.stop() - def durationInMillis = stopWatch.getTotalTimeMillis() + resourceMeter.stop() + def durationInMillis = resourceMeter.getTotalTimeMillis() then: 'insert new subscription with 1 second' - recordAndAssertPerformance("Insert new subscription", 1_000, durationInMillis) + recordAndAssertResourceUsage("Insert new subscription", 1_000, durationInMillis, 400,resourceMeter.getTotalMemoryUsageInMB()) } def querySubscriptionsByIteration(Collection allSubscriptionsAsDataNodes, targetSubscriptionSequenceNumber) { @@ -118,7 +118,7 @@ class CmDataSubscriptionsPerfTest extends NcmpPerfTestBase { String[] subscribersAsArray = it.leaves.get('subscribers') Set subscribersAsSet = new HashSet<>(Arrays.asList(subscribersAsArray)) def targetSubscriptionId = subscriberIdPrefix + '-' + ( targetSubscriptionSequenceNumber > 0 ? targetSubscriptionSequenceNumber - : 1 + random.nextInt(numberOfCmDataSubscribers) ) + : 1 + random.nextInt(numberOfCmDataSubscribers) ) if (subscribersAsSet.contains(targetSubscriptionId)) { matches.put(it, subscribersAsArray) } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy index 54e56d873..02881fec1 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy @@ -20,6 +20,7 @@ package org.onap.cps.integration.performance.ncmp +import org.onap.cps.integration.ResourceMeter import java.util.stream.Collectors import org.onap.cps.api.CpsQueryService import org.onap.cps.integration.performance.base.NcmpPerfTestBase @@ -29,22 +30,23 @@ import static org.onap.cps.spi.FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS class CmHandleQueryPerfTest extends NcmpPerfTestBase { CpsQueryService objectUnderTest + ResourceMeter resourceMeter = new ResourceMeter() def setup() { objectUnderTest = cpsQueryService } def 'Query CM Handle IDs by a property name and value.'() { when: 'a cps-path query on name-value pair is performed (without getting descendants)' - stopWatch.start() + resourceMeter.start() def cpsPath = '//additional-properties[@name="neType" and @value="RadioNode"]/ancestor::cm-handles' def dataNodes = objectUnderTest.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR, cpsPath, OMIT_DESCENDANTS) and: 'the ids of the result are extracted and converted to xpath' def xpaths = dataNodes.stream().map(dataNode -> "/dmi-registry/cm-handles[@id='${dataNode.leaves.id}']".toString() ).collect(Collectors.toSet()) and: 'a single get is executed to get all the parent objects and their descendants' def result = cpsDataService.getDataNodesForMultipleXpaths(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR, xpaths, INCLUDE_ALL_DESCENDANTS) - stopWatch.stop() - def durationInMillis = stopWatch.getTotalTimeMillis() + resourceMeter.stop() + def durationInMillis = resourceMeter.getTotalTimeMillis() then: 'the required operations are performed within 1200 ms' - recordAndAssertPerformance("CpsPath Registry attributes Query", 250, durationInMillis) + recordAndAssertResourceUsage("CpsPath Registry attributes Query", 250, durationInMillis, 150, resourceMeter.getTotalMemoryUsageInMB()) and: 'all but 1 (other node) are returned' result.size() == 999 and: 'the tree contains all the expected descendants too' diff --git a/integration-test/src/test/java/org/onap/cps/integration/ResourceMeter.java b/integration-test/src/test/java/org/onap/cps/integration/ResourceMeter.java new file mode 100644 index 000000000..c6ad96e91 --- /dev/null +++ b/integration-test/src/test/java/org/onap/cps/integration/ResourceMeter.java @@ -0,0 +1,70 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.integration; + +import org.springframework.util.StopWatch; + +/** + * Time and memory stop watch, exposing total running time and memory used. + */ +public class ResourceMeter { + private final StopWatch stopWatch = new StopWatch(); + private long memoryUsedBefore; + private long memoryUsedAfter; + + /** + * Start measurement. + */ + public void start() { + System.gc(); + memoryUsedBefore = getCurrentMemoryUsage(); + stopWatch.start(); + } + + /** + * Stop measurement. + */ + public void stop() { + stopWatch.stop(); + memoryUsedAfter = getCurrentMemoryUsage(); + } + + /** + * Get the total time in milliseconds. + * @return total time in milliseconds + */ + public long getTotalTimeMillis() { + return stopWatch.getTotalTimeMillis(); + } + + /** + * Get the total memory used in megabytes. + * @return total memory used in megabytes + */ + public double getTotalMemoryUsageInMB() { + return (memoryUsedAfter - memoryUsedBefore) / 1_000_000.0; + } + + private static long getCurrentMemoryUsage() { + return Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); + } +} +