From 2e07b499eb798c19c8641078ff79364f9439a281 Mon Sep 17 00:00:00 2001 From: danielhanrahan Date: Wed, 8 Mar 2023 13:50:34 +0000 Subject: [PATCH] Fetch fragment entities using recursive SQL query - Add SQL query that can fetch needed fragments to any given depth - Update getFragmentEntities to use new query - Remove now unused FragmentRepositoryMultiPathQuery - Remove unused TempTableCreator method - Result: getDataNodesForMultipleXpaths is up to 10 times faster Issue-ID: CPS-1525 Signed-off-by: danielhanrahan Change-Id: I07cbc9da5ab994ce7e0c2b02d7ca05089f05dab0 --- .../spi/impl/CpsDataPersistenceServiceImpl.java | 35 ++++++------- .../cps/spi/repository/FragmentRepository.java | 25 ++++++++- .../FragmentRepositoryMultiPathQuery.java | 31 ------------ .../FragmentRepositoryMultiPathQueryImpl.java | 59 ---------------------- .../onap/cps/spi/repository/TempTableCreator.java | 20 -------- ...CpsDataPersistenceServiceIntegrationSpec.groovy | 2 +- .../spi/impl/CpsDataPersistenceServiceSpec.groovy | 48 ++++++++++++------ .../CpsDataPersistenceServicePerfTest.groovy | 16 +++--- .../org/onap/cps/spi/FetchDescendantsOption.java | 8 +++ .../performance/base/CpsPerfTestBase.groovy | 22 ++++++++ .../integration/performance/cps/GetPerfTest.groovy | 8 +-- .../performance/ncmp/CmHandleQueryPerfTest.groovy | 16 +++--- 12 files changed, 122 insertions(+), 168 deletions(-) delete mode 100644 cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryMultiPathQuery.java delete mode 100644 cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryMultiPathQueryImpl.java diff --git a/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java b/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java index 3ea388242..82e6388d8 100644 --- a/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java @@ -256,12 +256,14 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService final Collection xpaths, final FetchDescendantsOption fetchDescendantsOption) { final AnchorEntity anchorEntity = getAnchorEntity(dataspaceName, anchorName); - final Collection fragmentEntities = getFragmentEntities(anchorEntity, xpaths); + final Collection fragmentEntities = + getFragmentEntities(anchorEntity, xpaths, fetchDescendantsOption); return toDataNodes(fragmentEntities, fetchDescendantsOption); } private Collection getFragmentEntities(final AnchorEntity anchorEntity, - final Collection xpaths) { + final Collection xpaths, + final FetchDescendantsOption fetchDescendantsOption) { final Collection nonRootXpaths = new HashSet<>(xpaths); final boolean haveRootXpath = nonRootXpaths.removeIf(CpsDataPersistenceServiceImpl::isRootXpath); @@ -273,15 +275,15 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService log.warn("Error parsing xpath \"{}\": {}", xpath, e.getMessage()); } } - final Collection fragmentEntities = - new HashSet<>(fragmentRepository.findByAnchorAndMultipleCpsPaths(anchorEntity.getId(), normalizedXpaths)); - if (haveRootXpath) { - final List fragmentExtracts = fragmentRepository.findAllExtractsByAnchor(anchorEntity); - fragmentEntities.addAll(FragmentEntityArranger.toFragmentEntityTrees(anchorEntity, fragmentExtracts)); + normalizedXpaths.addAll(fragmentRepository.findAllXpathByAnchorAndParentIdIsNull(anchorEntity)); } - return fragmentEntities; + final List fragmentExtracts = + fragmentRepository.findExtractsWithDescendants(anchorEntity.getId(), normalizedXpaths, + fetchDescendantsOption.getDepth()); + + return FragmentEntityArranger.toFragmentEntityTrees(anchorEntity, fragmentExtracts); } private FragmentEntity getFragmentEntity(final AnchorEntity anchorEntity, final String xpath) { @@ -325,7 +327,8 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService } fragmentEntities = fragmentRepository.findByAnchorAndCpsPath(anchorEntity.getId(), cpsPathQuery); if (cpsPathQuery.hasAncestorAxis()) { - fragmentEntities = getAncestorFragmentEntities(anchorEntity.getId(), cpsPathQuery, fragmentEntities); + final Collection ancestorXpaths = processAncestorXpath(fragmentEntities, cpsPathQuery); + fragmentEntities = getFragmentEntities(anchorEntity, ancestorXpaths, fetchDescendantsOption); } return createDataNodesFromProxiedFragmentEntities(fetchDescendantsOption, anchorEntity, fragmentEntities); } @@ -346,19 +349,12 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService fragmentRepository.quickFindWithDescendants(anchorEntity.getId(), xpathRegex); fragmentEntities = FragmentEntityArranger.toFragmentEntityTrees(anchorEntity, fragmentExtracts); if (cpsPathQuery.hasAncestorAxis()) { - fragmentEntities = getAncestorFragmentEntities(anchorEntity.getId(), cpsPathQuery, fragmentEntities); + final Collection ancestorXpaths = processAncestorXpath(fragmentEntities, cpsPathQuery); + fragmentEntities = getFragmentEntities(anchorEntity, ancestorXpaths, fetchDescendantsOption); } return createDataNodesFromFragmentEntities(fetchDescendantsOption, fragmentEntities); } - private Collection getAncestorFragmentEntities(final int anchorId, - final CpsPathQuery cpsPathQuery, - final Collection fragmentEntities) { - final Collection ancestorXpaths = processAncestorXpath(fragmentEntities, cpsPathQuery); - return ancestorXpaths.isEmpty() ? Collections.emptyList() - : fragmentRepository.findByAnchorAndMultipleCpsPaths(anchorId, ancestorXpaths); - } - private List createDataNodesFromProxiedFragmentEntities( final FetchDescendantsOption fetchDescendantsOption, final AnchorEntity anchorEntity, @@ -497,7 +493,8 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService .collect(Collectors.toMap(DataNode::getXpath, dataNode -> dataNode)); final Collection xpaths = xpathToUpdatedDataNode.keySet(); - final Collection existingFragmentEntities = getFragmentEntities(anchorEntity, xpaths); + final Collection existingFragmentEntities = + getFragmentEntities(anchorEntity, xpaths, FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS); for (final FragmentEntity existingFragmentEntity : existingFragmentEntities) { final DataNode updatedDataNode = xpathToUpdatedDataNode.get(existingFragmentEntity.getXpath()); diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java index 996e55e77..426a4601c 100755 --- a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java @@ -37,7 +37,7 @@ import org.springframework.stereotype.Repository; @Repository public interface FragmentRepository extends JpaRepository, FragmentRepositoryCpsPathQuery, - FragmentRepositoryMultiPathQuery, FragmentNativeRepository { + FragmentNativeRepository { Optional findByAnchorAndXpath(AnchorEntity anchorEntity, String xpath); @@ -68,9 +68,30 @@ public interface FragmentRepository extends JpaRepository, List quickFindWithDescendants(@Param("anchorId") int anchorId, @Param("xpathRegex") String xpathRegex); - @Query("SELECT xpath FROM FragmentEntity f WHERE anchor = :anchor AND xpath IN :xpaths") + @Query("SELECT xpath FROM FragmentEntity WHERE anchor = :anchor AND xpath IN :xpaths") List findAllXpathByAnchorAndXpathIn(@Param("anchor") AnchorEntity anchorEntity, @Param("xpaths") Collection xpaths); boolean existsByAnchorAndXpathStartsWith(AnchorEntity anchorEntity, String xpath); + + @Query("SELECT xpath FROM FragmentEntity WHERE anchor = :anchor AND parentId IS NULL") + List findAllXpathByAnchorAndParentIdIsNull(@Param("anchor") AnchorEntity anchorEntity); + + @Query(value + = "WITH RECURSIVE parent_search AS (" + + " SELECT id, 0 AS depth " + + " FROM fragment " + + " WHERE anchor_id = :anchorId AND xpath IN :xpaths " + + " UNION " + + " SELECT c.id, depth + 1 " + + " FROM fragment c INNER JOIN parent_search p ON c.parent_id = p.id" + + " WHERE depth <= (SELECT CASE WHEN :maxDepth = -1 THEN " + Integer.MAX_VALUE + " ELSE :maxDepth END) " + + ") " + + "SELECT f.id, anchor_id AS anchorId, xpath, f.parent_id AS parentId, CAST(attributes AS TEXT) AS attributes " + + "FROM fragment f INNER JOIN parent_search p ON f.id = p.id", + nativeQuery = true + ) + List findExtractsWithDescendants(@Param("anchorId") int anchorId, + @Param("xpaths") Collection xpaths, + @Param("maxDepth") int maxDepth); } diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryMultiPathQuery.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryMultiPathQuery.java deleted file mode 100644 index 9c34a459e..000000000 --- a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryMultiPathQuery.java +++ /dev/null @@ -1,31 +0,0 @@ -/*- - * ============LICENSE_START======================================================= - * Copyright (C) 2022 Nordix Foundation. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.onap.cps.spi.repository; - -import java.util.Collection; -import java.util.List; -import org.onap.cps.spi.entities.FragmentEntity; - -public interface FragmentRepositoryMultiPathQuery { - - List findByAnchorAndMultipleCpsPaths(Integer anchorId, Collection cpsPathQuery); - -} diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryMultiPathQueryImpl.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryMultiPathQueryImpl.java deleted file mode 100644 index 151fe97b3..000000000 --- a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryMultiPathQueryImpl.java +++ /dev/null @@ -1,59 +0,0 @@ -/*- - * ============LICENSE_START======================================================= - * Copyright (C) 2022-2023 Nordix Foundation. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.onap.cps.spi.repository; - -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import javax.persistence.EntityManager; -import javax.persistence.PersistenceContext; -import javax.transaction.Transactional; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.onap.cps.spi.entities.FragmentEntity; - -@Slf4j -@RequiredArgsConstructor -public class FragmentRepositoryMultiPathQueryImpl implements FragmentRepositoryMultiPathQuery { - - @PersistenceContext - private final EntityManager entityManager; - - private final TempTableCreator tempTableCreator; - - @Override - @Transactional - public List findByAnchorAndMultipleCpsPaths(final Integer anchorId, - final Collection cpsPathQueryList) { - if (cpsPathQueryList.isEmpty()) { - return Collections.emptyList(); - } - final String tempTableName = tempTableCreator.createTemporaryTable( - "xpathTemporaryTable", cpsPathQueryList, "xpath"); - final String sql = String.format( - "SELECT * FROM FRAGMENT WHERE anchor_id = %d AND xpath IN (select xpath FROM %s);", - anchorId, tempTableName); - final List fragmentEntities = entityManager.createNativeQuery(sql, FragmentEntity.class) - .getResultList(); - log.debug("Fetched {} fragment entities by anchor and cps path.", fragmentEntities.size()); - return fragmentEntities; - } -} diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/TempTableCreator.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/TempTableCreator.java index d798932c0..139a8b306 100644 --- a/cps-ri/src/main/java/org/onap/cps/spi/repository/TempTableCreator.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/TempTableCreator.java @@ -20,10 +20,8 @@ package org.onap.cps.spi.repository; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -66,24 +64,6 @@ public class TempTableCreator { return tempTableName; } - /** - * Create a uniquely named temporary table with a single column. - * - * @param prefix prefix for the table name (so you can recognize it) - * @param sqlData data to insert (strings only); each entry is a single row of data - * @param columnName column name - * @return a unique temporary table name with given prefix - */ - public String createTemporaryTable(final String prefix, - final Collection sqlData, - final String columnName) { - final Collection> tableData = new ArrayList<>(sqlData.size()); - for (final String entry : sqlData) { - tableData.add(Collections.singletonList(entry)); - } - return createTemporaryTable(prefix, tableData, columnName); - } - private static void defineColumns(final StringBuilder sqlStringBuilder, final String[] columnNames) { sqlStringBuilder.append('('); final Iterator it = Arrays.stream(columnNames).iterator(); diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceIntegrationSpec.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceIntegrationSpec.groovy index 25f19f7a8..e60afa78d 100755 --- a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceIntegrationSpec.groovy +++ b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceIntegrationSpec.groovy @@ -307,7 +307,7 @@ class CpsDataPersistenceServiceIntegrationSpec extends CpsPersistenceSpecBase { 'root xpath' | ["/"] || 7 'empty (root) xpath' | [""] || 7 'root and top-level xpaths' | ["/", "/parent-200", "/parent-201"] || 7 - 'root and child xpaths' | ["/", "/parent-200/child-201"] || 8 + 'root and child xpaths' | ["/", "/parent-200/child-201"] || 7 } @Sql([CLEAR_DATA, SET_DATA]) diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy index e74b4a745..3d7003d2a 100644 --- a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy +++ b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy @@ -26,6 +26,7 @@ import org.onap.cps.spi.FetchDescendantsOption import org.onap.cps.spi.entities.AnchorEntity import org.onap.cps.spi.entities.DataspaceEntity import org.onap.cps.spi.entities.FragmentEntity +import org.onap.cps.spi.entities.FragmentExtract import org.onap.cps.spi.exceptions.ConcurrencyException import org.onap.cps.spi.exceptions.DataValidationException import org.onap.cps.spi.model.DataNode @@ -150,9 +151,10 @@ class CpsDataPersistenceServiceSpec extends Specification { def 'Retrieving multiple data nodes.'() { given: 'fragment repository returns a collection of fragments' - def fragmentEntity1 = new FragmentEntity(xpath: '/xpath1', childFragments: []) - def fragmentEntity2 = new FragmentEntity(xpath: '/xpath2', childFragments: []) - mockFragmentRepository.findByAnchorAndMultipleCpsPaths(123, ['/xpath1', '/xpath2'] as Set) >> [fragmentEntity1, fragmentEntity2] + mockFragmentRepository.findExtractsWithDescendants(123, ['/xpath1', '/xpath2'] as Set, _) >> [ + mockFragmentExtract(1, null, 123, '/xpath1', null), + mockFragmentExtract(2, null, 123, '/xpath2', null) + ] when: 'getting data nodes for 2 xpaths' def result = objectUnderTest.getDataNodesForMultipleXpaths('some-dataspace', 'some-anchor', ['/xpath1', '/xpath2'], FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS) then: '2 data nodes are returned' @@ -203,8 +205,10 @@ class CpsDataPersistenceServiceSpec extends Specification { def 'update data node and descendants: #scenario'(){ given: 'the fragment repository returns fragment entities related to the xpath inputs' - mockFragmentRepository.findByAnchorAndMultipleCpsPaths(_, [] as Set) >> [] - mockFragmentRepository.findByAnchorAndMultipleCpsPaths(_, ['/test/xpath'] as Set) >> [new FragmentEntity(xpath: '/test/xpath', childFragments: [])] + mockFragmentRepository.findExtractsWithDescendants(_, [] as Set, _) >> [] + mockFragmentRepository.findExtractsWithDescendants(_, ['/test/xpath'] as Set, _) >> [ + mockFragmentExtract(1, null, 123, '/test/xpath', null) + ] when: 'replace data node tree' objectUnderTest.updateDataNodesAndDescendants('dataspaceName', 'anchorName', dataNodes) then: 'call fragment repository save all method' @@ -217,9 +221,10 @@ class CpsDataPersistenceServiceSpec extends Specification { def 'update data nodes and descendants'() { given: 'the fragment repository returns fragment entities related to the xpath inputs' - mockFragmentRepository.findByAnchorAndMultipleCpsPaths(_, ['/test/xpath1', '/test/xpath2'] as Set) >> [ - new FragmentEntity(xpath: '/test/xpath1', childFragments: [], anchor: anchorEntity), - new FragmentEntity(xpath: '/test/xpath2', childFragments: [], anchor: anchorEntity)] + mockFragmentRepository.findExtractsWithDescendants(123, ['/test/xpath1', '/test/xpath2'] as Set, _) >> [ + mockFragmentExtract(1, null, 123, '/test/xpath1', null), + mockFragmentExtract(2, null, 123, '/test/xpath2', null) + ] and: 'some data nodes with descendants' def dataNode1 = new DataNode(xpath: '/test/xpath1', leaves: ['id': 'testId1'], childDataNodes: [new DataNode(xpath: '/test/xpath1/child', leaves: ['id': 'childTestId1'])]) def dataNode2 = new DataNode(xpath: '/test/xpath2', leaves: ['id': 'testId2'], childDataNodes: [new DataNode(xpath: '/test/xpath2/child', leaves: ['id': 'childTestId2'])]) @@ -247,26 +252,39 @@ class CpsDataPersistenceServiceSpec extends Specification { def createDataNodesAndMockRepositoryMethodSupportingThem(Map xpathToScenarioMap) { def dataNodes = [] - def fragmentEntities = [] + def fragmentExtracts = [] + def fragmentId = 1 xpathToScenarioMap.each { def xpath = it.key def scenario = it.value def dataNode = new DataNodeBuilder().withXpath(xpath).build() dataNodes.add(dataNode) - def fragmentEntity = new FragmentEntity(xpath: xpath, childFragments: []) - fragmentEntities.add(fragmentEntity) - mockFragmentRepository.getByAnchorAndXpath(_, xpath) >> fragmentEntity + def fragmentExtract = mockFragmentExtract(fragmentId, null, null, xpath, null) + fragmentExtracts.add(fragmentExtract) + def fragmentEntity = new FragmentEntity(id: fragmentId, xpath: xpath, childFragments: []) + mockFragmentRepository.getByDataspaceAndAnchorAndXpath(_, _, xpath) >> fragmentEntity if ('EXCEPTION' == scenario) { mockFragmentRepository.save(fragmentEntity) >> { throw new StaleStateException("concurrent updates") } } + fragmentId++ } - mockFragmentRepository.findByAnchorAndMultipleCpsPaths(_, xpathToScenarioMap.keySet()) >> fragmentEntities + mockFragmentRepository.findExtractsWithDescendants(_, xpathToScenarioMap.keySet(), _) >> fragmentExtracts return dataNodes } def mockFragmentWithJson(json) { - def fragmentEntity = new FragmentEntity(xpath: '/parent-01', childFragments: [], attributes: json) - mockFragmentRepository.findByAnchorAndMultipleCpsPaths(123, ['/parent-01'] as Set) >> [fragmentEntity] + def fragmentExtract = mockFragmentExtract(456, null, null, '/parent-01', json) + mockFragmentRepository.findExtractsWithDescendants(123, ['/parent-01'] as Set, _) >> [fragmentExtract] + } + + def mockFragmentExtract(id, parentId, anchorId, xpath, attributes) { + def fragmentExtract = Mock(FragmentExtract) + fragmentExtract.getId() >> id + fragmentExtract.getParentId() >> parentId + fragmentExtract.getAnchorId() >> anchorId + fragmentExtract.getXpath() >> xpath + fragmentExtract.getAttributes() >> attributes + return fragmentExtract } } diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsDataPersistenceServicePerfTest.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsDataPersistenceServicePerfTest.groovy index 7f4716a7f..98ff211a6 100644 --- a/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsDataPersistenceServicePerfTest.groovy +++ b/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsDataPersistenceServicePerfTest.groovy @@ -72,8 +72,8 @@ class CpsDataPersistenceServicePerfTest extends CpsPersistencePerfSpecBase { assert countDataNodes(result[0]) == TOTAL_NUMBER_OF_NODES where: 'the following xPaths are used' scenario | xpath || allowedDuration - 'parent' | PERF_TEST_PARENT || 5000 - 'root' | '' || 500 + 'parent' | PERF_TEST_PARENT || 500 + 'root' | '/' || 500 } def 'Query parent data node with many descendants by cps-path'() { @@ -82,8 +82,8 @@ class CpsDataPersistenceServicePerfTest extends CpsPersistencePerfSpecBase { def result = objectUnderTest.queryDataNodes(PERF_DATASPACE, PERF_ANCHOR, '//perf-parent-1' , INCLUDE_ALL_DESCENDANTS) stopWatch.stop() def readDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'read duration is under 500 milliseconds' - recordAndAssertPerformance('Query with many descendants', 500, readDurationInMillis) + then: 'read duration is under 350 milliseconds' + recordAndAssertPerformance('Query with many descendants', 350, readDurationInMillis) and: 'data node is returned with all the descendants populated' assert countDataNodes(result) == TOTAL_NUMBER_OF_NODES } @@ -97,8 +97,8 @@ class CpsDataPersistenceServicePerfTest extends CpsPersistencePerfSpecBase { def readDurationInMillis = stopWatch.getTotalTimeMillis() then: 'the returned number of entities equal to the number of children * number of grandchildren' assert result.size() == xpathsToAllGrandChildren.size() - and: 'it took less then 5000ms' - recordAndAssertPerformance('Find multiple xpaths', 5000, readDurationInMillis) + and: 'it took less then 1000ms' + recordAndAssertPerformance('Find multiple xpaths', 1000, readDurationInMillis) } def 'Query many descendants by cps-path with #scenario'() { @@ -131,8 +131,8 @@ class CpsDataPersistenceServicePerfTest extends CpsPersistencePerfSpecBase { objectUnderTest.updateDataNodesAndDescendants(PERF_DATASPACE, PERF_ANCHOR, dataNodes) stopWatch.stop() def updateDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'update duration is under 900 milliseconds' - recordAndAssertPerformance('Update data nodes with descendants', 900, updateDurationInMillis) + then: 'update duration is under 500 milliseconds' + recordAndAssertPerformance('Update data nodes with descendants', 500, updateDurationInMillis) } def 'Update data nodes without descendants'() { diff --git a/cps-service/src/main/java/org/onap/cps/spi/FetchDescendantsOption.java b/cps-service/src/main/java/org/onap/cps/spi/FetchDescendantsOption.java index cf5e04dc4..02574995d 100644 --- a/cps-service/src/main/java/org/onap/cps/spi/FetchDescendantsOption.java +++ b/cps-service/src/main/java/org/onap/cps/spi/FetchDescendantsOption.java @@ -75,6 +75,14 @@ public class FetchDescendantsOption { return nextDescendantsOption; } + /** + * Get depth. + * @return depth: -1 for all descendants, 0 for no descendants, or positive value for fixed level of descendants + */ + public int getDepth() { + return depth; + } + /** * get fetch descendants option for given descendant. * diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy index 6fb6d844a..e75f1dce3 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy @@ -20,6 +20,8 @@ package org.onap.cps.integration.performance.base +import org.onap.cps.spi.FetchDescendantsOption + import java.time.OffsetDateTime import org.onap.cps.integration.base.CpsIntegrationSpecBase import org.onap.cps.rest.utils.MultipartFileUtil @@ -44,11 +46,21 @@ class CpsPerfTestBase extends PerfTestBase { } def createInitialData() { + createWarmupData() createLargeBookstoresData() addOpenRoadModel() addOpenRoadData() } + def createWarmupData() { + def data = "{\"bookstore\":{}}" + stopWatch.start() + addAnchorsWithData(1, CpsIntegrationSpecBase.BOOKSTORE_SCHEMA_SET, 'warmup', data) + stopWatch.stop() + def durationInMillis = stopWatch.getTotalTimeMillis() + recordAndAssertPerformance('Creating warmup anchor with tiny data tree', 250, durationInMillis) + } + def createLargeBookstoresData() { def data = CpsIntegrationSpecBase.readResourceDataFile('bookstore/largeModelData.json') stopWatch.start() @@ -80,7 +92,17 @@ class CpsPerfTestBase extends PerfTestBase { cpsAdminService.createAnchor(CPS_PERFORMANCE_TEST_DATASPACE, schemaSetName, anchorNamePrefix + it) cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, anchorNamePrefix + it, data, OffsetDateTime.now()) } + } + def 'Warm the database'() { + when: 'get data nodes for warmup anchor' + stopWatch.start() + def result = cpsDataService.getDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'warmup1', '/', FetchDescendantsOption.OMIT_DESCENDANTS) + assert countDataNodesInTree(result) == 1 + stopWatch.stop() + def durationInMillis = stopWatch.getTotalTimeMillis() + then: 'all data is read within 15 seconds (warm up not critical)' + recordAndAssertPerformance("Warming database", 15_000, durationInMillis) } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy index 753faf44f..30e8bf23d 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy @@ -42,10 +42,10 @@ class GetPerfTest extends CpsPerfTestBase { recordAndAssertPerformance("Read datatrees using ${scenario}", durationLimit, durationInMillis) where: 'the following xpaths are used' scenario | anchorPrefix | xpath || durationLimit | expectedNumberOfDataNodes - 'bookstore root' | 'bookstore' | '/' || 25_000 | 78 - 'bookstore top element' | 'bookstore' | '/bookstore' || 1_000 | 78 - 'openroadm root' | 'openroadm' | '/' || 1_000 | 2151 - 'openroadm top element' | 'openroadm' | '/openroadm-devices' || 10_000 | 2151 + 'bookstore root' | 'bookstore' | '/' || 130 | 78 + 'bookstore top element' | 'bookstore' | '/bookstore' || 130 | 78 + 'openroadm root' | 'openroadm' | '/' || 750 | 2151 + 'openroadm top element' | 'openroadm' | '/openroadm-devices' || 750 | 2151 } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy index 939281a73..87327030c 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy @@ -21,12 +21,11 @@ package org.onap.cps.integration.performance.ncmp import java.util.stream.Collectors - +import org.onap.cps.integration.performance.base.NcmpRegistryPerfTestBase +import org.springframework.dao.DataAccessResourceFailureException import static org.onap.cps.spi.FetchDescendantsOption.OMIT_DESCENDANTS import static org.onap.cps.spi.FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS -import org.onap.cps.integration.performance.base.NcmpRegistryPerfTestBase - class CmHandleQueryPerfTest extends NcmpRegistryPerfTestBase { def objectUnderTest @@ -52,14 +51,13 @@ class CmHandleQueryPerfTest extends NcmpRegistryPerfTestBase { assert countDataNodesInTree(result) == 5 * 999 } - def 'Multiple get limitation: 32,764 (~ 2^15) xpaths.'() { + def 'Multiple get limit exceeded: 32,764 (~ 2^15) xpaths.'() { given: 'more than 32,764 xpaths)' - def xpaths = [] - (0..32_765).each { xpaths.add("/size/of/this/path/does/not/matter/for/limit[@id='" + it + "']") } - when: 'get single get is executed to get all the parent objects and their descendants' + def xpaths = (0..32_764).collect(i -> "/size/of/this/path/does/not/matter/for/limit[@id='" + i + "']") + when: 'single get is executed to get all the parent objects and their descendants' cpsDataService.getDataNodesForMultipleXpaths(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR, xpaths, INCLUDE_ALL_DESCENDANTS) - then: 'no exception is thrown (limit is not present in current implementation)' - noExceptionThrown() + then: 'an exception is thrown' + thrown(DataAccessResourceFailureException.class) } } -- 2.16.6