/*
* ============LICENSE_START=======================================================
* Copyright (C) 2021 highstreet technologies GmbH
- * Modifications Copyright (C) 2021-2022 Nordix Foundation
+ * Modifications Copyright (C) 2021-2023 Nordix Foundation
* Modifications Copyright (C) 2021 Pantheon.tech
* Modifications Copyright (C) 2021-2022 Bell Canada
* ================================================================================
import static org.onap.cps.ncmp.api.impl.operations.DmiRequestBody.OperationEnum;
import static org.onap.cps.ncmp.api.impl.utils.RestQueryParametersValidator.validateCmHandleQueryParameters;
+import com.google.common.collect.Lists;
import com.hazelcast.map.IMap;
import java.time.OffsetDateTime;
import java.util.ArrayList;
@RequiredArgsConstructor
public class NetworkCmProxyDataServiceImpl implements NetworkCmProxyDataService {
+ private static final int DELETE_BATCH_SIZE = 100;
private final JsonObjectMapper jsonObjectMapper;
private final DmiDataOperations dmiDataOperations;
private final NetworkCmProxyDataServicePropertyHandler networkCmProxyDataServicePropertyHandler;
.collect(Collectors.toList());
updateCmHandleStateBatch(yangModelCmHandles, CmHandleState.DELETING);
- for (final String cmHandleId : tobeRemovedCmHandles) {
+ for (final List<String> tobeRemovedCmHandleBatch : Lists.partition(tobeRemovedCmHandles, DELETE_BATCH_SIZE)) {
try {
- deleteCmHandleFromDbAndModuleSyncMap(cmHandleId);
- cmHandleRegistrationResponses.add(CmHandleRegistrationResponse.createSuccessResponse(cmHandleId));
- } catch (final DataNodeNotFoundException dataNodeNotFoundException) {
- log.error("Unable to find dataNode for cmHandleId : {} , caused by : {}",
- cmHandleId, dataNodeNotFoundException.getMessage());
- cmHandleRegistrationResponses.add(CmHandleRegistrationResponse
- .createFailureResponse(cmHandleId, RegistrationError.CM_HANDLE_DOES_NOT_EXIST));
- } catch (final DataValidationException dataValidationException) {
- log.error("Unable to de-register cm-handle id: {}, caused by: {}",
- cmHandleId, dataValidationException.getMessage());
- cmHandleRegistrationResponses.add(CmHandleRegistrationResponse
- .createFailureResponse(cmHandleId, RegistrationError.CM_HANDLE_INVALID_ID));
- } catch (final Exception exception) {
- log.error("Unable to de-register cm-handle id : {} , caused by : {}",
- cmHandleId, exception.getMessage());
- cmHandleRegistrationResponses.add(
- CmHandleRegistrationResponse.createFailureResponse(cmHandleId, exception));
+ batchDeleteCmHandlesFromDbAndModuleSyncMap(tobeRemovedCmHandleBatch);
+ tobeRemovedCmHandleBatch.forEach(cmHandleId ->
+ cmHandleRegistrationResponses.add(CmHandleRegistrationResponse.createSuccessResponse(cmHandleId)));
+
+ } catch (final Exception batchException) {
+ log.error("Unable to de-register cm-handle batch, retrying on each cm handle");
+ for (final String cmHandleId : tobeRemovedCmHandleBatch) {
+ final CmHandleRegistrationResponse cmHandleRegistrationResponse =
+ deleteCmHandleAndGetCmHandleRegistrationResponse(cmHandleId);
+ cmHandleRegistrationResponses.add(cmHandleRegistrationResponse);
+ }
}
}
return cmHandleRegistrationResponses;
}
+ private CmHandleRegistrationResponse deleteCmHandleAndGetCmHandleRegistrationResponse(final String cmHandleId) {
+ try {
+ deleteCmHandleFromDbAndModuleSyncMap(cmHandleId);
+ return CmHandleRegistrationResponse.createSuccessResponse(cmHandleId);
+ } catch (final DataNodeNotFoundException dataNodeNotFoundException) {
+ log.error("Unable to find dataNode for cmHandleId : {} , caused by : {}",
+ cmHandleId, dataNodeNotFoundException.getMessage());
+ return CmHandleRegistrationResponse.createFailureResponse(cmHandleId,
+ RegistrationError.CM_HANDLE_DOES_NOT_EXIST);
+ } catch (final DataValidationException dataValidationException) {
+ log.error("Unable to de-register cm-handle id: {}, caused by: {}",
+ cmHandleId, dataValidationException.getMessage());
+ return CmHandleRegistrationResponse.createFailureResponse(cmHandleId,
+ RegistrationError.CM_HANDLE_INVALID_ID);
+ } catch (final Exception exception) {
+ log.error("Unable to de-register cm-handle id : {} , caused by : {}", cmHandleId, exception.getMessage());
+ return CmHandleRegistrationResponse.createFailureResponse(cmHandleId, exception);
+ }
+ }
+
private void updateCmHandleStateBatch(final List<YangModelCmHandle> yangModelCmHandles,
final CmHandleState cmHandleState) {
final Map<YangModelCmHandle, CmHandleState> cmHandleIdsToBeRemoved = new HashMap<>();
private void deleteCmHandleFromDbAndModuleSyncMap(final String cmHandleId) {
inventoryPersistence.deleteSchemaSetWithCascade(cmHandleId);
- inventoryPersistence.deleteListOrListElement("/dmi-registry/cm-handles[@id='" + cmHandleId + "']");
+ inventoryPersistence.deleteDataNode("/dmi-registry/cm-handles[@id='" + cmHandleId + "']");
removeDeletedCmHandleFromModuleSyncMap(cmHandleId);
}
+ private void batchDeleteCmHandlesFromDbAndModuleSyncMap(final Collection<String> tobeRemovedCmHandles) {
+ tobeRemovedCmHandles.forEach(inventoryPersistence::deleteSchemaSetWithCascade);
+ inventoryPersistence.deleteDataNodes(mapCmHandleIdsToXpaths(tobeRemovedCmHandles));
+ tobeRemovedCmHandles.forEach(this::removeDeletedCmHandleFromModuleSyncMap);
+ }
+
+ private Collection<String> mapCmHandleIdsToXpaths(final Collection<String> cmHandles) {
+ return cmHandles.stream()
+ .map(cmHandleId -> "/dmi-registry/cm-handles[@id='" + cmHandleId + "']")
+ .collect(Collectors.toSet());
+ }
+
// CPS-1239 Robustness cleaning of in progress cache
private void removeDeletedCmHandleFromModuleSyncMap(final String deletedCmHandleId) {
if (moduleSyncStartedOnCmHandles.remove(deletedCmHandleId) != null) {
/*
* ============LICENSE_START=======================================================
- * Copyright (C) 2022 Nordix Foundation
+ * Copyright (C) 2022-2023 Nordix Foundation
* ================================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
void replaceListContent(String parentNodeXpath, Collection<DataNode> dataNodes);
/**
- * Deletes data node for given anchor and dataspace.
+ * Deletes data node.
*
* @param dataNodeXpath data node xpath
*/
void deleteDataNode(String dataNodeXpath);
+
+ /**
+ * Deletes multiple data nodes.
+ *
+ * @param dataNodeXpaths data node xpaths
+ */
+ void deleteDataNodes(Collection<String> dataNodeXpaths);
}
@Override
public void deleteDataNode(final String dataNodeXpath) {
- cpsDataService.deleteDataNode(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR, dataNodeXpath,
- NO_TIMESTAMP);
+ cpsDataService.deleteDataNode(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR, dataNodeXpath, NO_TIMESTAMP);
+ }
+
+ @Override
+ public void deleteDataNodes(final Collection<String> dataNodeXpaths) {
+ cpsDataService.deleteDataNodes(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR, dataNodeXpaths, NO_TIMESTAMP);
}
}
/*
* ============LICENSE_START=======================================================
- * Copyright (C) 2021-2022 Nordix Foundation
+ * Copyright (C) 2021-2023 Nordix Foundation
* Modifications Copyright (C) 2022 Bell Canada
* ================================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
and: 'method to delete relevant schema set is called once'
1 * mockInventoryPersistence.deleteSchemaSetWithCascade(_)
and: 'method to delete relevant list/list element is called once'
- 1 * mockInventoryPersistence.deleteListOrListElement(_)
+ 1 * mockInventoryPersistence.deleteDataNodes(_)
and: 'successful response is received'
assert response.getRemovedCmHandles().size() == 1
with(response.getRemovedCmHandles().get(0)) {
given: 'a registration with three cm-handles to be deleted'
def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server',
removedCmHandles: ['cmhandle1', 'cmhandle2', 'cmhandle3'])
+ and: 'cm-handle deletion fails on batch'
+ mockInventoryPersistence.deleteDataNodes(_) >> { throw new RuntimeException("Failed") }
and: 'cm-handle deletion is successful for 1st and 3rd; failed for 2nd'
- mockInventoryPersistence.deleteListOrListElement(_) >> {} >> { throw new RuntimeException("Failed") } >> {}
+ mockInventoryPersistence.deleteDataNode(_) >> {} >> { throw new RuntimeException("Failed") } >> {}
when: 'registration is updated to delete cmhandles'
def response = objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
then: 'a response is received for all cm-handles'
then: 'no exception is thrown'
noExceptionThrown()
and: 'cm-handle is not deleted'
- 0 * mockInventoryPersistence.deleteListOrListElement(_)
+ 0 * mockInventoryPersistence.deleteDataNodes(_)
and: 'the cmHandle state is not updated to "DELETED"'
0 * mockLcmEventsCmHandleStateHandler.updateCmHandleState(_, CmHandleState.DELETED)
and: 'a failure response is received'
def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server',
removedCmHandles: ['cmhandle'])
and: 'cm-handle deletion throws exception'
- mockInventoryPersistence.deleteListOrListElement(_) >> { throw deleteListElementException }
+ mockInventoryPersistence.deleteDataNodes(_) >> { throw deleteListElementException }
+ mockInventoryPersistence.deleteDataNode(_) >> { throw deleteListElementException }
when: 'registration is updated to delete cmhandle'
def response = objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration)
then: 'a failure response is received'
'sample dataNode xpath', NO_TIMESTAMP);
}
+ def 'Delete multiple data nodes via xPath'() {
+ when: 'Delete data nodes method is called with multiple xpaths as parameters'
+ objectUnderTest.deleteDataNodes(['xpath1', 'xpath2'])
+ then: 'the cps data service method to delete data nodes is invoked once with the same xPaths'
+ 1 * mockCpsDataService.deleteDataNodes('NCMP-Admin', 'ncmp-dmi-registry',
+ ['xpath1', 'xpath2'], NO_TIMESTAMP);
+ }
+
}
import org.onap.cps.spi.model.DataNodeBuilder;
import org.onap.cps.spi.repository.AnchorRepository;
import org.onap.cps.spi.repository.DataspaceRepository;
-import org.onap.cps.spi.repository.FragmentNativeRepository;
import org.onap.cps.spi.repository.FragmentQueryBuilder;
import org.onap.cps.spi.repository.FragmentRepository;
import org.onap.cps.spi.utils.SessionManager;
private final FragmentRepository fragmentRepository;
private final JsonObjectMapper jsonObjectMapper;
private final SessionManager sessionManager;
- private final FragmentNativeRepository fragmentNativeRepositoryImpl;
private static final String REG_EX_FOR_OPTIONAL_LIST_INDEX = "(\\[@[\\s\\S]+?]){0,1})";
anchorEntity -> fragmentRepository.deleteByAnchorIn(Set.of(anchorEntity)));
}
+ @Override
+ @Transactional
+ public void deleteDataNodes(final String dataspaceName, final String anchorName,
+ final Collection<String> xpathsToDelete) {
+ final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName);
+ final AnchorEntity anchorEntity = anchorRepository.getByDataspaceAndName(dataspaceEntity, anchorName);
+
+ final Collection<String> normalizedXpaths = new ArrayList<>(xpathsToDelete.size());
+ for (final String xpath : xpathsToDelete) {
+ try {
+ normalizedXpaths.add(CpsPathUtil.getNormalizedXpath(xpath));
+ } catch (final PathParsingException e) {
+ log.debug("Error parsing xpath \"{}\" in deleteDataNodes: {}", xpath, e.getMessage());
+ }
+ }
+
+ fragmentRepository.deleteByAnchorIdAndXpaths(anchorEntity.getId(), normalizedXpaths);
+ fragmentRepository.deleteListsByAnchorIdAndXpaths(anchorEntity.getId(), normalizedXpaths);
+ }
+
@Override
@Transactional
public void deleteListDataNode(final String dataspaceName, final String anchorName,
private boolean deleteDataNode(final FragmentEntity parentFragmentEntity, final String targetXpath) {
final String normalizedTargetXpath = CpsPathUtil.getNormalizedXpath(targetXpath);
if (parentFragmentEntity.getXpath().equals(normalizedTargetXpath)) {
- fragmentNativeRepositoryImpl.deleteFragmentEntity(parentFragmentEntity.getId());
+ fragmentRepository.deleteFragmentEntity(parentFragmentEntity.getId());
return true;
}
if (parentFragmentEntity.getChildFragments()
package org.onap.cps.spi.repository;
+import java.util.Collection;
+
/**
* This interface is used in delete fragment entity by id with child using native sql queries.
*/
public interface FragmentNativeRepository {
void deleteFragmentEntity(long fragmentEntityId);
+
+ /**
+ * Delete fragment entities for each supplied xpath.
+ * This method will delete list elements or other data nodes, but not whole lists.
+ * Non-existing xpaths will not result in an exception.
+ * @param anchorId the id of the anchor
+ * @param xpaths xpaths of data nodes to remove
+ */
+ void deleteByAnchorIdAndXpaths(int anchorId, Collection<String> xpaths);
+
+ /**
+ * Delete fragment entities that are list elements of each supplied list xpath.
+ * For example, if xpath '/parent/list' is provided, then list all elements in '/parent/list' will be deleted,
+ * e.g. /parent/list[@key='A'], /parent/list[@key='B'].
+ * This method will only delete whole lists by xpath; xpaths to list elements or other data nodes will be ignored.
+ * Non-existing xpaths will not result in an exception.
+ * @param anchorId the id of the anchor
+ * @param listXpaths xpaths of whole lists to remove
+ */
+ void deleteListsByAnchorIdAndXpaths(int anchorId, Collection<String> listXpaths);
}
package org.onap.cps.spi.repository;
-import java.sql.PreparedStatement;
+import java.util.Collection;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
-import org.hibernate.Session;
-import org.springframework.stereotype.Repository;
+import lombok.RequiredArgsConstructor;
-@Repository
+@RequiredArgsConstructor
public class FragmentNativeRepositoryImpl implements FragmentNativeRepository {
private static final String DROP_FRAGMENT_CONSTRAINT
private static final String ADD_FRAGMENT_CONSTRAINT_WITH_CASCADE
= "ALTER TABLE fragment ADD CONSTRAINT fragment_parent_id_fkey FOREIGN KEY (parent_id) "
+ "REFERENCES fragment (id) ON DELETE CASCADE;";
- private static final String DELETE_FRAGMENT = "DELETE FROM fragment WHERE id =?;";
private static final String ADD_ORIGINAL_FRAGMENT_CONSTRAINT
= "ALTER TABLE fragment ADD CONSTRAINT fragment_parent_id_fkey FOREIGN KEY (parent_id) "
+ "REFERENCES fragment (id) ON DELETE NO ACTION;";
@PersistenceContext
- private EntityManager entityManager;
+ private final EntityManager entityManager;
+
+ private final TempTableCreator tempTableCreator;
@Override
public void deleteFragmentEntity(final long fragmentEntityId) {
- final Session session = entityManager.unwrap(Session.class);
- session.doWork(connection -> {
- try (PreparedStatement preparedStatement = connection.prepareStatement(
+ entityManager.createNativeQuery(
+ DROP_FRAGMENT_CONSTRAINT
+ + ADD_FRAGMENT_CONSTRAINT_WITH_CASCADE
+ + "DELETE FROM fragment WHERE id = ?;"
+ + DROP_FRAGMENT_CONSTRAINT
+ + ADD_ORIGINAL_FRAGMENT_CONSTRAINT)
+ .setParameter(1, fragmentEntityId)
+ .executeUpdate();
+ }
+
+ @Override
+ // Accept security hotspot as temporary table name in SQL query is created internally, not from user input.
+ @SuppressWarnings("squid:S2077")
+ public void deleteByAnchorIdAndXpaths(final int anchorId, final Collection<String> xpaths) {
+ if (!xpaths.isEmpty()) {
+ final String tempTableName = tempTableCreator.createTemporaryTable("xpathsToDelete", xpaths, "xpath");
+ entityManager.createNativeQuery(
DROP_FRAGMENT_CONSTRAINT
- + ADD_FRAGMENT_CONSTRAINT_WITH_CASCADE
- + DELETE_FRAGMENT
- + DROP_FRAGMENT_CONSTRAINT
- + ADD_ORIGINAL_FRAGMENT_CONSTRAINT)) {
- preparedStatement.setLong(1, fragmentEntityId);
- preparedStatement.executeUpdate();
- }
- });
+ + ADD_FRAGMENT_CONSTRAINT_WITH_CASCADE
+ + "DELETE FROM fragment f USING " + tempTableName + " t"
+ + " WHERE f.anchor_id = :anchorId AND f.xpath = t.xpath;"
+ + DROP_FRAGMENT_CONSTRAINT
+ + ADD_ORIGINAL_FRAGMENT_CONSTRAINT)
+ .setParameter("anchorId", anchorId)
+ .executeUpdate();
+ }
}
-}
+ @Override
+ // Accept security hotspot as temporary table name in SQL query is created internally, not from user input.
+ @SuppressWarnings("squid:S2077")
+ public void deleteListsByAnchorIdAndXpaths(final int anchorId, final Collection<String> xpaths) {
+ if (!xpaths.isEmpty()) {
+ final String tempTableName = tempTableCreator.createTemporaryTable("xpathsToDelete", xpaths, "xpath");
+ entityManager.createNativeQuery(
+ DROP_FRAGMENT_CONSTRAINT
+ + ADD_FRAGMENT_CONSTRAINT_WITH_CASCADE
+ + "DELETE FROM fragment f USING " + tempTableName + " t"
+ + " WHERE f.anchor_id = :anchorId AND f.xpath LIKE CONCAT(t.xpath, :xpathListPattern);"
+ + DROP_FRAGMENT_CONSTRAINT
+ + ADD_ORIGINAL_FRAGMENT_CONSTRAINT)
+ .setParameter("anchorId", anchorId)
+ .setParameter("xpathListPattern", "[%%")
+ .executeUpdate();
+ }
+ }
+
+}
/*\r
* ============LICENSE_START=======================================================\r
- * Copyright (C) 2021-2022 Nordix Foundation.\r
+ * Copyright (C) 2021-2023 Nordix Foundation.\r
* Modifications Copyright (C) 2020-2021 Bell Canada.\r
* Modifications Copyright (C) 2020-2021 Pantheon.tech.\r
* ================================================================================\r
\r
@Repository\r
public interface FragmentRepository extends JpaRepository<FragmentEntity, Long>, FragmentRepositoryCpsPathQuery,\r
- FragmentRepositoryMultiPathQuery {\r
+ FragmentRepositoryMultiPathQuery, FragmentNativeRepository {\r
\r
Optional<FragmentEntity> findByDataspaceAndAnchorAndXpath(@NonNull DataspaceEntity dataspaceEntity,\r
@NonNull AnchorEntity anchorEntity,\r
package org.onap.cps.spi.repository;
-
-import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.HashSet;
import java.util.List;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import javax.transaction.Transactional;
-import lombok.AllArgsConstructor;
+import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.onap.cps.spi.entities.FragmentEntity;
-
@Slf4j
-@AllArgsConstructor
+@RequiredArgsConstructor
public class FragmentRepositoryMultiPathQueryImpl implements FragmentRepositoryMultiPathQuery {
@PersistenceContext
- private EntityManager entityManager;
+ private final EntityManager entityManager;
- private TempTableCreator tempTableCreator;
+ private final TempTableCreator tempTableCreator;
@Override
@Transactional
if (cpsPathQueryList.isEmpty()) {
return Collections.emptyList();
}
- final Collection<List<String>> sqlData = new HashSet<>(cpsPathQueryList.size());
- for (final String query : cpsPathQueryList) {
- final List<String> row = new ArrayList<>(1);
- row.add(query);
- sqlData.add(row);
- }
-
final String tempTableName = tempTableCreator.createTemporaryTable(
- "xpathTemporaryTable", sqlData, "xpath");
- return selectMatchingFragments(anchorId, tempTableName);
- }
-
- private List<FragmentEntity> selectMatchingFragments(final Integer anchorId, final String tempTableName) {
+ "xpathTemporaryTable", cpsPathQueryList, "xpath");
final String sql = String.format(
"SELECT * FROM FRAGMENT WHERE anchor_id = %d AND xpath IN (select xpath FROM %s);",
anchorId, tempTableName);
final List<FragmentEntity> fragmentEntities = entityManager.createNativeQuery(sql, FragmentEntity.class)
- .getResultList();
+ .getResultList();
log.debug("Fetched {} fragment entities by anchor and cps path.", fragmentEntities.size());
return fragmentEntities;
}
/*-
* ============LICENSE_START=======================================================
- * Copyright (C) 2022 Nordix Foundation.
+ * Copyright (C) 2022-2023 Nordix Foundation.
* ================================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
package org.onap.cps.spi.repository;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
return tempTableName;
}
+ /**
+ * Create a uniquely named temporary table with a single column.
+ *
+ * @param prefix prefix for the table name (so you can recognize it)
+ * @param sqlData data to insert (strings only); each entry is a single row of data
+ * @param columnName column name
+ * @return a unique temporary table name with given prefix
+ */
+ public String createTemporaryTable(final String prefix,
+ final Collection<String> sqlData,
+ final String columnName) {
+ final Collection<List<String>> tableData = new ArrayList<>(sqlData.size());
+ for (final String entry : sqlData) {
+ tableData.add(Collections.singletonList(entry));
+ }
+ return createTemporaryTable(prefix, tableData, columnName);
+ }
+
private static void defineColumns(final StringBuilder sqlStringBuilder, final String[] columnNames) {
sqlStringBuilder.append('(');
final Iterator<String> it = Arrays.stream(columnNames).iterator();
def 'Delete list scenario: #scenario.'() {
when: 'deleting list is executed for: #scenario.'
objectUnderTest.deleteListDataNode(DATASPACE_NAME, ANCHOR_NAME3, targetXpaths)
- then: 'only the expected children remain'
+ and: 'remaining children are fetched'
def parentFragment = fragmentRepository.getById(parentFragmentId)
def remainingChildXpaths = parentFragment.childFragments.collect { it.xpath }
+ then: 'only the expected children remain'
assert remainingChildXpaths.size() == expectedRemainingChildXpaths.size()
assert remainingChildXpaths.containsAll(expectedRemainingChildXpaths)
where: 'following parameters were used'
'list element under list element' | '/parent-203/child-204[@key="B"]/grand-child-204[@key2="Y"]' | LIST_DATA_NODE_PARENT203_FRAGMENT_ID || ["/parent-203/child-203", "/parent-203/child-204[@key='A']", "/parent-203/child-204[@key='B']"]
}
+ @Sql([CLEAR_DATA, SET_DATA])
+ def 'Delete multiple data nodes using scenario: #scenario.'() {
+ when: 'deleting nodes is executed for: #scenario.'
+ objectUnderTest.deleteDataNodes(DATASPACE_NAME, ANCHOR_NAME3, targetXpaths)
+ and: 'remaining children are fetched'
+ def parentFragment = fragmentRepository.getById(LIST_DATA_NODE_PARENT203_FRAGMENT_ID)
+ def remainingChildXpaths = parentFragment.childFragments.collect { it.xpath }
+ then: 'only the expected children remain'
+ assert remainingChildXpaths.size() == expectedRemainingChildXpaths.size()
+ assert remainingChildXpaths.containsAll(expectedRemainingChildXpaths)
+ where: 'following parameters were used'
+ scenario | targetXpaths || expectedRemainingChildXpaths
+ 'delete nothing' | [] || ["/parent-203/child-203", "/parent-203/child-204[@key='A']", "/parent-203/child-204[@key='B']"]
+ 'datanode' | ['/parent-203/child-203'] || ["/parent-203/child-204[@key='A']", "/parent-203/child-204[@key='B']"]
+ '1 list element' | ['/parent-203/child-204[@key="A"]'] || ["/parent-203/child-203", "/parent-203/child-204[@key='B']"]
+ '2 list elements' | ['/parent-203/child-204[@key="A"]', '/parent-203/child-204[@key="B"]'] || ["/parent-203/child-203"]
+ 'whole list' | ['/parent-203/child-204'] || ['/parent-203/child-203']
+ 'list and element in same list' | ['/parent-203/child-204', '/parent-203/child-204[@key="A"]'] || ['/parent-203/child-203']
+ 'list element under list element' | ['/parent-203/child-204[@key="B"]/grand-child-204[@key2="Y"]'] || ["/parent-203/child-203", "/parent-203/child-204[@key='A']", "/parent-203/child-204[@key='B']"]
+ 'valid but non-existing xpath' | ['/non-existing', '/parent-203/child-204'] || ['/parent-203/child-203']
+ 'invalid xpath' | ['INVALID XPATH', '/parent-203/child-204'] || ['/parent-203/child-203']
+ }
+
@Sql([CLEAR_DATA, SET_DATA])
def 'Delete data nodes with "/"-token in list key value: #scenario. (CPS-1409)'() {
given: 'a data nodes with list-element child with "/" in index value (and grandchild)'
import org.onap.cps.spi.model.DataNodeBuilder
import org.onap.cps.spi.repository.AnchorRepository
import org.onap.cps.spi.repository.DataspaceRepository
-import org.onap.cps.spi.repository.FragmentNativeRepository
import org.onap.cps.spi.repository.FragmentRepository
import org.onap.cps.spi.utils.SessionManager
import org.onap.cps.utils.JsonObjectMapper
def mockFragmentRepository = Mock(FragmentRepository)
def jsonObjectMapper = new JsonObjectMapper(new ObjectMapper())
def mockSessionManager = Mock(SessionManager)
- def stubFragmentNativeRepository = Stub(FragmentNativeRepository)
def objectUnderTest = Spy(new CpsDataPersistenceServiceImpl(mockDataspaceRepository, mockAnchorRepository,
- mockFragmentRepository, jsonObjectMapper, mockSessionManager, stubFragmentNativeRepository))
+ mockFragmentRepository, jsonObjectMapper, mockSessionManager))
def 'Storing data nodes individually when batch operation fails'(){
given: 'two data nodes and supporting repository mock behavior'
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.test.context.jdbc.Sql
import org.springframework.util.StopWatch
-import spock.lang.Shared
-
-import java.util.concurrent.TimeUnit
class CpsDataPersistenceServiceDeletePerfTest extends CpsPersistencePerfSpecBase {
@Autowired
CpsDataPersistenceService objectUnderTest
- static def NUMBER_OF_CHILDREN = 100
- static def NUMBER_OF_GRAND_CHILDREN = 50
- static def NUMBER_OF_LISTS = 100
- static def NUMBER_OF_LIST_ELEMENTS = 50
- static def ALLOWED_SETUP_TIME_MS = TimeUnit.SECONDS.toMillis(10)
-
def stopWatch = new StopWatch()
@Sql([CLEAR_DATA, PERF_TEST_DATA])
def 'Create a node with many descendants (please note, subsequent tests depend on this running first).'() {
when: 'a node with a large number of descendants is created'
stopWatch.start()
- createLineage(objectUnderTest, NUMBER_OF_CHILDREN, NUMBER_OF_GRAND_CHILDREN, false)
+ createLineage(objectUnderTest, 150, 50, false)
stopWatch.stop()
def setupDurationInMillis = stopWatch.getTotalTimeMillis()
- then: 'setup duration is under #ALLOWED_SETUP_TIME_MS milliseconds'
- recordAndAssertPerformance('Setup',ALLOWED_SETUP_TIME_MS, setupDurationInMillis)
+ then: 'setup duration is under 10 seconds'
+ recordAndAssertPerformance('Setup', 10_000, setupDurationInMillis)
}
def 'Delete 5 children with grandchildren'() {
when: 'child nodes are deleted'
stopWatch.start()
(1..5).each {
- def childPath = "${PERF_TEST_PARENT}/perf-test-child-${it}".toString();
+ def childPath = "${PERF_TEST_PARENT}/perf-test-child-${it}".toString()
objectUnderTest.deleteDataNode(PERF_DATASPACE, PERF_ANCHOR, childPath)
}
stopWatch.stop()
def deleteDurationInMillis = stopWatch.getTotalTimeMillis()
- then: 'delete duration is under 300 milliseconds'
- recordAndAssertPerformance('Delete 5 children', 300, deleteDurationInMillis)
+ then: 'delete duration is under 350 milliseconds'
+ recordAndAssertPerformance('Delete 5 children', 350, deleteDurationInMillis)
+ }
+
+ def 'Batch delete 100 children with grandchildren'() {
+ given: 'a list of xpaths to delete'
+ def xpathsToDelete = (6..105).collect {
+ "${PERF_TEST_PARENT}/perf-test-child-${it}".toString()
+ }
+ when: 'child nodes are deleted'
+ stopWatch.start()
+ objectUnderTest.deleteDataNodes(PERF_DATASPACE, PERF_ANCHOR, xpathsToDelete)
+ stopWatch.stop()
+ def deleteDurationInMillis = stopWatch.getTotalTimeMillis()
+ then: 'delete duration is under 350 milliseconds'
+ recordAndAssertPerformance('Batch delete 100 children', 350, deleteDurationInMillis)
}
def 'Delete 50 grandchildren (that have no descendants)'() {
when: 'target nodes are deleted'
stopWatch.start()
(1..50).each {
- def grandchildPath = "${PERF_TEST_PARENT}/perf-test-child-6/perf-test-grand-child-${it}".toString();
+ def grandchildPath = "${PERF_TEST_PARENT}/perf-test-child-106/perf-test-grand-child-${it}".toString()
objectUnderTest.deleteDataNode(PERF_DATASPACE, PERF_ANCHOR, grandchildPath)
}
stopWatch.stop()
recordAndAssertPerformance('Delete 50 grandchildren', 350, deleteDurationInMillis)
}
- def 'Delete 1 large data node with many descendants'() {
- when: 'parent node is deleted'
+ def 'Batch delete 500 grandchildren (that have no descendants)'() {
+ given: 'a list of xpaths to delete'
+ def xpathsToDelete = []
+ for (int childIndex = 0; childIndex < 10; childIndex++) {
+ xpathsToDelete.addAll((1..50).collect {
+ "${PERF_TEST_PARENT}/perf-test-child-${107+childIndex}/perf-test-grand-child-${it}".toString()
+ })
+ }
+ when: 'target nodes are deleted'
stopWatch.start()
- objectUnderTest.deleteDataNode(PERF_DATASPACE, PERF_ANCHOR, PERF_TEST_PARENT)
+ objectUnderTest.deleteDataNodes(PERF_DATASPACE, PERF_ANCHOR, xpathsToDelete)
stopWatch.stop()
def deleteDurationInMillis = stopWatch.getTotalTimeMillis()
- then: 'delete duration is under 250 milliseconds'
- recordAndAssertPerformance('Delete one large node', 250, deleteDurationInMillis)
+ then: 'delete duration is under 350 milliseconds'
+ recordAndAssertPerformance('Batch delete 500 grandchildren', 350, deleteDurationInMillis)
}
@Sql([CLEAR_DATA, PERF_TEST_DATA])
def 'Create a node with many list elements (please note, subsequent tests depend on this running first).'() {
- given: 'a node with a large number of descendants is created'
+ when: 'a node with a large number of lists is created'
stopWatch.start()
- createLineage(objectUnderTest, NUMBER_OF_LISTS, NUMBER_OF_LIST_ELEMENTS, true)
+ createLineage(objectUnderTest, 150, 50, true)
stopWatch.stop()
def setupDurationInMillis = stopWatch.getTotalTimeMillis()
- and: 'setup duration is under #ALLOWED_SETUP_TIME_MS milliseconds'
- recordAndAssertPerformance('Create node with many list elements', ALLOWED_SETUP_TIME_MS, setupDurationInMillis)
+ then: 'setup duration is under 10 seconds'
+ recordAndAssertPerformance('Setup lists', 10_000, setupDurationInMillis)
}
- def 'Delete 5 whole lists with many elements'() {
- when: 'list nodes are deleted'
+ def 'Delete 5 whole lists'() {
+ when: 'lists are deleted'
stopWatch.start()
(1..5).each {
- def childPath = "${PERF_TEST_PARENT}/perf-test-list-${it}".toString();
+ def childPath = "${PERF_TEST_PARENT}/perf-test-list-${it}".toString()
objectUnderTest.deleteListDataNode(PERF_DATASPACE, PERF_ANCHOR, childPath)
}
stopWatch.stop()
def deleteDurationInMillis = stopWatch.getTotalTimeMillis()
- then: 'delete duration is under 1000 milliseconds'
+ then: 'delete duration is under 1500 milliseconds'
recordAndAssertPerformance('Delete 5 whole lists', 1500, deleteDurationInMillis)
}
- def 'Delete 10 list elements with keys'() {
+ def 'Batch delete 100 whole lists'() {
+ given: 'a list of xpaths to delete'
+ def xpathsToDelete = (6..105).collect {
+ "${PERF_TEST_PARENT}/perf-test-list-${it}".toString()
+ }
+ when: 'lists are deleted'
+ stopWatch.start()
+ objectUnderTest.deleteDataNodes(PERF_DATASPACE, PERF_ANCHOR, xpathsToDelete)
+ stopWatch.stop()
+ def deleteDurationInMillis = stopWatch.getTotalTimeMillis()
+ then: 'delete duration is under 350 milliseconds'
+ recordAndAssertPerformance('Batch delete 100 whole lists', 350, deleteDurationInMillis)
+ }
+
+ def 'Delete 10 list elements'() {
when: 'list elements are deleted'
stopWatch.start()
(1..10).each {
- def key = it.toString()
- def grandchildPath = "${PERF_TEST_PARENT}/perf-test-list-6[@key='${key}']"
+ def grandchildPath = "${PERF_TEST_PARENT}/perf-test-list-106[@key='${it}']".toString()
objectUnderTest.deleteListDataNode(PERF_DATASPACE, PERF_ANCHOR, grandchildPath)
}
stopWatch.stop()
def deleteDurationInMillis = stopWatch.getTotalTimeMillis()
- then: 'delete duration is under 1200 milliseconds'
- recordAndAssertPerformance('Delete 10 lists elements', 1500, deleteDurationInMillis)
+ then: 'delete duration is under 750 milliseconds'
+ recordAndAssertPerformance('Delete 10 lists elements', 750, deleteDurationInMillis)
+ }
+
+ def 'Batch delete 500 list elements'() {
+ given: 'a list of xpaths to delete'
+ def xpathsToDelete = []
+ for (int childIndex = 0; childIndex < 10; childIndex++) {
+ xpathsToDelete.addAll((1..50).collect {
+ "${PERF_TEST_PARENT}/perf-test-list-${107+childIndex}[@key='${it}']".toString()
+ })
+ }
+ when: 'list elements are deleted'
+ stopWatch.start()
+ objectUnderTest.deleteDataNodes(PERF_DATASPACE, PERF_ANCHOR, xpathsToDelete)
+ stopWatch.stop()
+ def deleteDurationInMillis = stopWatch.getTotalTimeMillis()
+ then: 'delete duration is under 350 milliseconds'
+ recordAndAssertPerformance('Batch delete 500 lists elements', 350, deleteDurationInMillis)
+ }
+
+ @Sql([CLEAR_DATA, PERF_TEST_DATA])
+ def 'Delete 1 large data node'() {
+ given: 'a node with a large number of descendants is created'
+ createLineage(objectUnderTest, 50, 50, false)
+ createLineage(objectUnderTest, 50, 50, true)
+ when: 'parent node is deleted'
+ stopWatch.start()
+ objectUnderTest.deleteDataNode(PERF_DATASPACE, PERF_ANCHOR, PERF_TEST_PARENT)
+ stopWatch.stop()
+ def deleteDurationInMillis = stopWatch.getTotalTimeMillis()
+ then: 'delete duration is under 300 milliseconds'
+ recordAndAssertPerformance('Delete one large node', 300, deleteDurationInMillis)
+ }
+
+ @Sql([CLEAR_DATA, PERF_TEST_DATA])
+ def 'Batch delete 1 large data node'() {
+ given: 'a node with a large number of descendants is created'
+ createLineage(objectUnderTest, 50, 50, false)
+ createLineage(objectUnderTest, 50, 50, true)
+ when: 'parent node is batch deleted'
+ stopWatch.start()
+ objectUnderTest.deleteDataNodes(PERF_DATASPACE, PERF_ANCHOR, [PERF_TEST_PARENT])
+ stopWatch.stop()
+ def deleteDurationInMillis = stopWatch.getTotalTimeMillis()
+ then: 'delete duration is under 300 milliseconds'
+ recordAndAssertPerformance('Batch delete one large node', 300, deleteDurationInMillis)
}
@Sql([CLEAR_DATA, PERF_TEST_DATA])
def 'Delete root node with many descendants'() {
given: 'a node with a large number of descendants is created'
- createLineage(objectUnderTest, NUMBER_OF_CHILDREN, NUMBER_OF_GRAND_CHILDREN, false)
+ createLineage(objectUnderTest, 50, 50, false)
+ createLineage(objectUnderTest, 50, 50, true)
when: 'root node is deleted'
stopWatch.start()
objectUnderTest.deleteDataNode(PERF_DATASPACE, PERF_ANCHOR, '/')
stopWatch.stop()
def deleteDurationInMillis = stopWatch.getTotalTimeMillis()
- then: 'delete duration is under 250 milliseconds'
- recordAndAssertPerformance('Delete root node', 250, deleteDurationInMillis)
+ then: 'delete duration is under 300 milliseconds'
+ recordAndAssertPerformance('Delete root node', 300, deleteDurationInMillis)
}
@Sql([CLEAR_DATA, PERF_TEST_DATA])
- def 'Delete data nodes for an anchor'() {
+ def 'Delete data nodes for an anchor'() {212
given: 'a node with a large number of descendants is created'
- createLineage(objectUnderTest, NUMBER_OF_CHILDREN, NUMBER_OF_GRAND_CHILDREN, false)
+ createLineage(objectUnderTest, 50, 50, false)
+ createLineage(objectUnderTest, 50, 50, true)
when: 'data nodes are deleted'
stopWatch.start()
objectUnderTest.deleteDataNodes(PERF_DATASPACE, PERF_ANCHOR)
stopWatch.stop()
def deleteDurationInMillis = stopWatch.getTotalTimeMillis()
- then: 'delete duration is under 250 milliseconds'
- recordAndAssertPerformance('Delete data nodes for anchor', 250, deleteDurationInMillis)
+ then: 'delete duration is under 300 milliseconds'
+ recordAndAssertPerformance('Delete data nodes for anchor', 300, deleteDurationInMillis)
}
}
/*
* ============LICENSE_START=======================================================
- * Copyright (C) 2020-2022 Nordix Foundation
+ * Copyright (C) 2020-2023 Nordix Foundation
* Modifications Copyright (C) 2021 Pantheon.tech
* Modifications Copyright (C) 2021-2022 Bell Canada
* Modifications Copyright (C) 2022 Deutsche Telekom AG
/**
* Deletes data node for given anchor and dataspace.
*
- * @param dataspaceName dataspace name
- * @param anchorName anchor name
- * @param dataNodeXpath data node xpath
+ * @param dataspaceName dataspace name
+ * @param anchorName anchor name
+ * @param dataNodeXpath data node xpath
* @param observedTimestamp observed timestamp
*/
void deleteDataNode(String dataspaceName, String anchorName, String dataNodeXpath,
OffsetDateTime observedTimestamp);
+ /**
+ * Deletes multiple data nodes for given anchor and dataspace.
+ *
+ * @param dataspaceName dataspace name
+ * @param anchorName anchor name
+ * @param dataNodeXpaths data node xpaths
+ * @param observedTimestamp observed timestamp
+ */
+ void deleteDataNodes(String dataspaceName, String anchorName, Collection<String> dataNodeXpaths,
+ OffsetDateTime observedTimestamp);
+
/**
* Deletes all data nodes for a given anchor in a dataspace.
*
* @param dataspaceName dataspace name
- * @param anchorName anchor name
+ * @param anchorName anchor name
* @param observedTimestamp observed timestamp
*/
void deleteDataNodes(String dataspaceName, String anchorName, OffsetDateTime observedTimestamp);
@Timed(value = "cps.data.service.datanode.batch.delete",
description = "Time taken to delete a batch of datanodes")
public void deleteDataNodes(final String dataspaceName, final String anchorName,
- final OffsetDateTime observedTimestamp) {
+ final Collection<String> dataNodeXpaths, final OffsetDateTime observedTimestamp) {
+ cpsValidator.validateNameCharacters(dataspaceName, anchorName);
+ cpsDataPersistenceService.deleteDataNodes(dataspaceName, anchorName, dataNodeXpaths);
+ dataNodeXpaths.forEach(dataNodeXpath ->
+ processDataUpdatedEventAsync(dataspaceName, anchorName, dataNodeXpath, DELETE, observedTimestamp));
+ }
+
+ @Override
+ @Timed(value = "cps.data.service.datanode.all.delete",
+ description = "Time taken to delete all datanodes")
+ public void deleteDataNodes(final String dataspaceName, final String anchorName,
+ final OffsetDateTime observedTimestamp) {
cpsValidator.validateNameCharacters(dataspaceName, anchorName);
processDataUpdatedEventAsync(dataspaceName, anchorName, ROOT_NODE_XPATH, DELETE, observedTimestamp);
cpsDataPersistenceService.deleteDataNodes(dataspaceName, anchorName);
/*
* ============LICENSE_START=======================================================
- * Copyright (C) 2020-2022 Nordix Foundation.
+ * Copyright (C) 2020-2023 Nordix Foundation.
* Modifications Copyright (C) 2021 Pantheon.tech
* Modifications Copyright (C) 2022 Bell Canada
* Modifications Copyright (C) 2022 TechMahindra Ltd.
*/
void deleteDataNode(String dataspaceName, String anchorName, String targetXpath);
+ /**
+ * Deletes multiple dataNode, yang container or yang list or yang list element.
+ *
+ * @param dataspaceName dataspace name
+ * @param anchorName anchor name
+ * @param targetXpaths xpaths of nodes to delete
+ */
+ void deleteDataNodes(String dataspaceName, String anchorName, Collection<String> targetXpaths);
+
/**
* Deletes all dataNodes in a given anchor.
*
void deleteDataNodes(String dataspaceName, String anchorName);
/**
- * Deletes existing a single list element or the whole list.
+ * Deletes a single existing list element or the whole list.
*
* @param dataspaceName dataspace name
* @param anchorName anchor name
1 * mockNotificationService.processDataUpdatedEvent(dataspaceName, anchorName, '/test-tree/branch', Operation.DELETE, observedTimestamp)
}
+ def 'Delete multiple list elements under existing node.'() {
+ given: 'schema set for given anchor and dataspace references test-tree model'
+ setupSchemaSetMocks('test-tree.yang')
+ when: 'delete multiple list data method is invoked with list element json data'
+ objectUnderTest.deleteDataNodes(dataspaceName, anchorName, ['/test-tree/branch[@name="A"]', '/test-tree/branch[@name="B"]'], observedTimestamp)
+ then: 'the persistence service method is invoked with correct parameters'
+ 1 * mockCpsDataPersistenceService.deleteDataNodes(dataspaceName, anchorName, ['/test-tree/branch[@name="A"]', '/test-tree/branch[@name="B"]'])
+ and: 'the CpsValidator is called on the dataspaceName and AnchorName'
+ 1 * mockCpsValidator.validateNameCharacters(dataspaceName, anchorName)
+ and: 'two data updated events are sent to notification service'
+ 2 * mockNotificationService.processDataUpdatedEvent(dataspaceName, anchorName, _, Operation.DELETE, observedTimestamp)
+ }
+
def 'Delete data node under anchor and dataspace.'() {
given: 'schema set for given anchor and dataspace references test tree model'
setupSchemaSetMocks('test-tree.yang')
export $(cut -d= -f1 $WORKSPACE/plans/cps/test.properties)
###################### setup cps-ncmp ############################
-mkdir -p $WORKSPACE/archives/dc-cps
-cp $WORKSPACE/../docker-compose/*.yml $WORKSPACE/archives/dc-cps
-cd $WORKSPACE/archives/dc-cps
+cd $CPS_HOME/docker-compose
curl -L https://github.com/docker/compose/releases/download/1.29.2/docker-compose-`uname -s`-`uname -m` > docker-compose
chmod +x docker-compose
ADVISED_MODULES_SYNC_SLEEP_TIME_MS=2000
CMHANDLE_DATA_SYNC_SLEEP_TIME_MS=2000
+
+CPS_HOME=$CPS_HOME
\ No newline at end of file
# $1 test options (passed on to run-csit.sh as such)
export TESTOPTIONS=${1}
+export CPS_HOME=$(git rev-parse --show-toplevel)
export WORKSPACE=$(git rev-parse --show-toplevel)/csit
rm -rf ${WORKSPACE}/archives
restart: unless-stopped
init-db:
- build: ../../../docker-compose/initfile
+ build: ${CPS_HOME:-.}/docker-compose/initfile
environment:
CPS_CORE_PORT: ${CPS_CORE_PORT:-8883}
CPS_CORE_MANAGEMENT_PORT: ${CPS_CORE_MANAGEMENT_PORT:-8887}
volumes:
- - ./initfile:/initfile
- - ../../../cps-ncmp-service/src/main/resources/model/:/model
+ - ${CPS_HOME:-.}/cps-ncmp-service/src/main/resources/model/:/model
depends_on:
cps-and-ncmp:
condition: service_started
import org.onap.cps.spi.impl.CpsModulePersistenceServiceImpl
import org.onap.cps.spi.repository.AnchorRepository
import org.onap.cps.spi.repository.DataspaceRepository
-import org.onap.cps.spi.repository.FragmentNativeRepository
import org.onap.cps.spi.repository.FragmentRepository
import org.onap.cps.spi.repository.ModuleReferenceRepository
import org.onap.cps.spi.repository.SchemaSetRepository
@Lazy
ModuleReferenceRepository moduleReferenceRepository
- @Autowired
- @Lazy
- FragmentNativeRepository fragmentNativeRepository
-
@Autowired
@Lazy
JsonObjectMapper jsonObjectMapper
@Bean
CpsDataPersistenceService cpsDataPersistenceService() {
- return (CpsDataPersistenceService) new CpsDataPersistenceServiceImpl(dataspaceRepository, anchorRepository, fragmentRepository, jsonObjectMapper, stubbedSessionManager, fragmentNativeRepository)
+ return (CpsDataPersistenceService) new CpsDataPersistenceServiceImpl(dataspaceRepository, anchorRepository, fragmentRepository, jsonObjectMapper, stubbedSessionManager)
}
@Bean
Code Review / cps.git / commitdiff