- Bulk delete anchors and datanodes associated with schemasets.
Improves de-registration performance by approx 10%
Issue-ID: CPS-1423
Signed-off-by: danielhanrahan <daniel.hanrahan@est.tech>
Change-Id: Ie38e8b4c64356bf5935d8c7a5d3f5bfa73fb1714
/*
* ============LICENSE_START=======================================================
- * Copyright (C) 2020-2022 Nordix Foundation.
+ * Copyright (C) 2020-2023 Nordix Foundation.
* Modifications Copyright (C) 2020-2022 Bell Canada.
* Modifications Copyright (C) 2021 Pantheon.tech
* Modifications Copyright (C) 2022 TechMahindra Ltd.
.collect(Collectors.toSet());
}
+ @Override
+ public Collection<Anchor> getAnchors(final String dataspaceName, final Collection<String> schemaSetNames) {
+ final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName);
+ return anchorRepository.findAllByDataspaceAndSchemaSetNameIn(dataspaceEntity, schemaSetNames)
+ .stream().map(CpsAdminPersistenceServiceImpl::toAnchor).collect(Collectors.toSet());
+ }
+
@Override
public Collection<Anchor> queryAnchors(final String dataspaceName, final Collection<String> inputModuleNames) {
try {
anchorRepository.delete(anchorEntity);
}
+ @Transactional
+ @Override
+ public void deleteAnchors(final String dataspaceName, final Collection<String> anchorNames) {
+ final var dataspaceEntity = dataspaceRepository.getByName(dataspaceName);
+ anchorRepository.deleteAllByDataspaceAndNameIn(dataspaceEntity, anchorNames);
+ }
+
private AnchorEntity getAnchorEntity(final String dataspaceName, final String anchorName) {
final var dataspaceEntity = dataspaceRepository.getByName(dataspaceName);
return anchorRepository.getByDataspaceAndName(dataspaceEntity, anchorName);
anchorEntity -> fragmentRepository.deleteByAnchorIn(Set.of(anchorEntity)));
}
+ @Override
+ @Transactional
+ public void deleteDataNodes(final String dataspaceName, final Collection<String> anchorNames) {
+ final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName);
+ final Collection<AnchorEntity> anchorEntities =
+ anchorRepository.findAllByDataspaceAndNameIn(dataspaceEntity, anchorNames);
+ fragmentRepository.deleteByAnchorIn(anchorEntities);
+ }
+
@Override
@Transactional
public void deleteDataNodes(final String dataspaceName, final String anchorName,
Collection<AnchorEntity> findAllBySchemaSet(@NotNull SchemaSetEntity schemaSetEntity);
+ Collection<AnchorEntity> findAllByDataspaceAndNameIn(@NotNull DataspaceEntity dataspaceEntity,
+ @NotNull Collection<String> anchorNames);
+
+ Collection<AnchorEntity> findAllByDataspaceAndSchemaSetNameIn(@NotNull DataspaceEntity dataspaceEntity,
+ @NotNull Collection<String> schemaSetNames);
+
Integer countByDataspace(@NotNull DataspaceEntity dataspaceEntity);
@Query(value = "SELECT anchor.* FROM yang_resource\n"
+ "HAVING COUNT(DISTINCT module_name) = :sizeOfModuleNames", nativeQuery = true)
Collection<AnchorEntity> getAnchorsByDataspaceIdAndModuleNames(@Param("dataspaceId") int dataspaceId,
@Param("moduleNames") Collection<String> moduleNames, @Param("sizeOfModuleNames") int sizeOfModuleNames);
-}
\ No newline at end of file
+
+ void deleteAllByDataspaceAndNameIn(@NotNull DataspaceEntity dataspaceEntity,
+ @NotNull Collection<String> anchorNames);
+}
/*
* ============LICENSE_START=======================================================
- * Copyright (C) 2021-2022 Nordix Foundation
+ * Copyright (C) 2021-2023 Nordix Foundation
* Modifications Copyright (C) 2021 Pantheon.tech
* Modifications Copyright (C) 2022 Bell Canada
* Modifications Copyright (C) 2022 TechMahindra Ltd.
where: 'the following data is used'
dataspaceName || expectedAnchors
DATASPACE_NAME || [Anchor.builder().name(ANCHOR_NAME1).schemaSetName(SCHEMA_SET_NAME1).dataspaceName(DATASPACE_NAME).build(),
- Anchor.builder().name(ANCHOR_NAME2).schemaSetName(SCHEMA_SET_NAME2).dataspaceName(DATASPACE_NAME).build()]
+ Anchor.builder().name(ANCHOR_NAME2).schemaSetName(SCHEMA_SET_NAME2).dataspaceName(DATASPACE_NAME).build(),
+ Anchor.builder().name(ANCHOR_NAME3).schemaSetName(SCHEMA_SET_NAME2).dataspaceName(DATASPACE_NAME).build()]
DATASPACE_WITH_NO_DATA || []
}
thrown(DataspaceNotFoundException)
}
+ @Sql([CLEAR_DATA, SET_DATA])
+ def 'Get all anchors associated with multiple schemasets in a dataspace.'() {
+ when: 'anchors are retrieved by dataspace and schema-sets'
+ def anchors = objectUnderTest.getAnchors('DATASPACE-001', ['SCHEMA-SET-001', 'SCHEMA-SET-002'])
+ then: ' the response contains expected anchors'
+ anchors == Set.of(
+ new Anchor('ANCHOR-001', 'DATASPACE-001', 'SCHEMA-SET-001'),
+ new Anchor('ANCHOR-002', 'DATASPACE-001', 'SCHEMA-SET-002'),
+ new Anchor('ANCHOR-003', 'DATASPACE-001', 'SCHEMA-SET-002'))
+ }
+
@Sql([CLEAR_DATA, SET_DATA])
def 'Delete anchor'() {
when: 'delete anchor action is invoked'
'anchor does not exists' | DATASPACE_NAME | 'unknown' || AnchorNotFoundException
}
+ @Sql([CLEAR_DATA, SET_DATA])
+ def 'Delete multiple anchors'() {
+ when: 'delete anchors action is invoked'
+ objectUnderTest.deleteAnchors(DATASPACE_NAME, ['ANCHOR-002', 'ANCHOR-003'])
+ then: 'anchors are deleted'
+ anchorRepository.findById(3002).isEmpty()
+ anchorRepository.findById(3003).isEmpty()
+ }
+
@Sql([CLEAR_DATA, SAMPLE_DATA_FOR_ANCHORS_WITH_MODULES])
def 'Query anchors that have #scenario.'() {
when: 'all anchor are retrieved for the given dataspace name and module names'
where: 'the following data is used'
scenario | dataspaceName || expectedException | expectedMessageDetails
'dataspace name does not exist' | 'unknown' || DataspaceNotFoundException | 'unknown does not exist'
- 'dataspace contains an anchor' | 'DATASPACE-001' || DataspaceInUseException | 'contains 2 anchor(s)'
+ 'dataspace contains an anchor' | 'DATASPACE-001' || DataspaceInUseException | 'contains 3 anchor(s)'
'dataspace contains schemasets' | 'DATASPACE-003' || DataspaceInUseException | 'contains 1 schemaset(s)'
}
}
static DataNodeBuilder dataNodeBuilder = new DataNodeBuilder()
static final String SET_DATA = '/data/fragment.sql'
- static int DATASPACE_1001_ID = 1001L
- static int ANCHOR_3003_ID = 3003L
static long ID_DATA_NODE_WITH_DESCENDANTS = 4001
static String XPATH_DATA_NODE_WITH_DESCENDANTS = '/parent-1'
static String XPATH_DATA_NODE_WITH_LEAVES = '/parent-207'
@Sql([CLEAR_DATA, SET_DATA])
def 'Delete data node for an anchor.'() {
given: 'a data-node exists for an anchor'
- assert fragmentsExistInDB(DATASPACE_1001_ID, ANCHOR_3003_ID)
+ assert fragmentsExistInDB(1001, 3003)
when: 'data nodes are deleted '
objectUnderTest.deleteDataNodes(DATASPACE_NAME, ANCHOR_NAME3)
then: 'all data-nodes are deleted successfully'
- assert !fragmentsExistInDB(DATASPACE_1001_ID, ANCHOR_3003_ID)
+ assert !fragmentsExistInDB(1001, 3003)
+ }
+
+ @Sql([CLEAR_DATA, SET_DATA])
+ def 'Delete data node for multiple anchors.'() {
+ given: 'a data-node exists for an anchor'
+ assert fragmentsExistInDB(1001, 3001)
+ assert fragmentsExistInDB(1001, 3003)
+ when: 'data nodes are deleted '
+ objectUnderTest.deleteDataNodes(DATASPACE_NAME, ['ANCHOR-001', 'ANCHOR-003'])
+ then: 'all data-nodes are deleted successfully'
+ assert !fragmentsExistInDB(1001, 3001)
+ assert !fragmentsExistInDB(1001, 3003)
}
def fragmentsExistInDB(dataSpaceId, anchorId) {
}
@Sql([CLEAR_DATA, PERF_TEST_DATA])
- def 'Delete data nodes for an anchor'() {212
+ def 'Delete data nodes for an anchor'() {
given: 'a node with a large number of descendants is created'
createLineage(objectUnderTest, 50, 50, false)
createLineage(objectUnderTest, 50, 50, true)
recordAndAssertPerformance('Delete data nodes for anchor', 300, deleteDurationInMillis)
}
+ @Sql([CLEAR_DATA, PERF_TEST_DATA])
+ def 'Delete data nodes for multiple anchors'() {
+ given: 'a node with a large number of descendants is created'
+ createLineage(objectUnderTest, 50, 50, false)
+ createLineage(objectUnderTest, 50, 50, true)
+ when: 'data nodes are deleted'
+ stopWatch.start()
+ objectUnderTest.deleteDataNodes(PERF_DATASPACE, [PERF_ANCHOR])
+ stopWatch.stop()
+ def deleteDurationInMillis = stopWatch.getTotalTimeMillis()
+ then: 'delete duration is under 300 milliseconds'
+ recordAndAssertPerformance('Delete data nodes for anchor', 300, deleteDurationInMillis)
+ }
+
}
/*
============LICENSE_START=======================================================
Copyright (C) 2020 Pantheon.tech
- Modifications Copyright (C) 2020 Nordix Foundation.
+ Modifications Copyright (C) 2020-2023 Nordix Foundation.
Modifications Copyright (C) 2021-2022 Bell Canada.
================================================================================
Licensed under the Apache License, Version 2.0 (the "License");
INSERT INTO ANCHOR (ID, NAME, DATASPACE_ID, SCHEMA_SET_ID) VALUES
(3001, 'ANCHOR-001', 1001, 2001),
- (3002, 'ANCHOR-002', 1001, 2002);
+ (3002, 'ANCHOR-002', 1001, 2002),
+ (3003, 'ANCHOR-003', 1001, 2002);
INSERT INTO FRAGMENT (ID, DATASPACE_ID, ANCHOR_ID, PARENT_ID, XPATH, ATTRIBUTES) VALUES
(4001, 1001, 3001, null, '/xpath', '{}');
/*
* ============LICENSE_START=======================================================
- * Copyright (C) 2020-2022 Nordix Foundation
+ * Copyright (C) 2020-2023 Nordix Foundation
* Modifications Copyright (C) 2020-2022 Bell Canada.
* Modifications Copyright (C) 2021 Pantheon.tech
* Modifications Copyright (C) 2022 TechMahindra Ltd.
Collection<Anchor> getAnchors(String dataspaceName);
/**
- * Read all anchors associated the given schema-set in the given dataspace.
+ * Read all anchors associated with the given schema-set in the given dataspace.
*
* @param dataspaceName dataspace name
* @param schemaSetName schema-set name
*/
Collection<Anchor> getAnchors(String dataspaceName, String schemaSetName);
+ /**
+ * Read all anchors associated with the given schema-sets in the given dataspace.
+ *
+ * @param dataspaceName dataspace name
+ * @param schemaSetNames schema-set names
+ * @return a collection of anchors
+ */
+ Collection<Anchor> getAnchors(String dataspaceName, Collection<String> schemaSetNames);
+
/**
* Get an anchor in the given dataspace using the anchor name.
*
*/
void deleteAnchor(String dataspaceName, String anchorName);
+ /**
+ * Delete anchors by name in given dataspace.
+ *
+ * @param dataspaceName dataspace name
+ * @param anchorNames anchor names
+ */
+ void deleteAnchors(String dataspaceName, Collection<String> anchorNames);
+
/**
* Query anchor names for the given module names in the provided dataspace.
*
*/
void deleteDataNodes(String dataspaceName, String anchorName, OffsetDateTime observedTimestamp);
+ /**
+ * Deletes all data nodes for multiple anchors in a dataspace.
+ *
+ * @param dataspaceName dataspace name
+ * @param anchorNames anchor names
+ * @param observedTimestamp observed timestamp
+ */
+ void deleteDataNodes(String dataspaceName, Collection<String> anchorNames, OffsetDateTime observedTimestamp);
+
/**
* Deletes a list or a list-element under given anchor and dataspace.
*
/*
* ============LICENSE_START=======================================================
- * Copyright (C) 2020-2022 Nordix Foundation
+ * Copyright (C) 2020-2023 Nordix Foundation
* Modifications Copyright (C) 2020-2022 Bell Canada.
* Modifications Copyright (C) 2021 Pantheon.tech
* Modifications Copyright (C) 2022 TechMahindra Ltd.
return cpsAdminPersistenceService.getAnchors(dataspaceName, schemaSetName);
}
+ @Override
+ public Collection<Anchor> getAnchors(final String dataspaceName, final Collection<String> schemaSetNames) {
+ cpsValidator.validateNameCharacters(dataspaceName);
+ cpsValidator.validateNameCharacters(schemaSetNames);
+ return cpsAdminPersistenceService.getAnchors(dataspaceName, schemaSetNames);
+ }
+
@Override
public Anchor getAnchor(final String dataspaceName, final String anchorName) {
cpsValidator.validateNameCharacters(dataspaceName, anchorName);
cpsAdminPersistenceService.deleteAnchor(dataspaceName, anchorName);
}
+ @Override
+ public void deleteAnchors(final String dataspaceName, final Collection<String> anchorNames) {
+ cpsValidator.validateNameCharacters(dataspaceName);
+ cpsValidator.validateNameCharacters(anchorNames);
+ cpsDataService.deleteDataNodes(dataspaceName, anchorNames, OffsetDateTime.now());
+ cpsAdminPersistenceService.deleteAnchors(dataspaceName, anchorNames);
+ }
+
@Override
public Collection<String> queryAnchorNames(final String dataspaceName, final Collection<String> moduleNames) {
cpsValidator.validateNameCharacters(dataspaceName);
}
@Override
- @Timed(value = "cps.data.service.datanode.all.delete",
- description = "Time taken to delete all datanodes")
+ @Timed(value = "cps.data.service.datanode.delete.anchor",
+ description = "Time taken to delete all datanodes for an anchor")
public void deleteDataNodes(final String dataspaceName, final String anchorName,
final OffsetDateTime observedTimestamp) {
cpsValidator.validateNameCharacters(dataspaceName, anchorName);
cpsDataPersistenceService.deleteDataNodes(dataspaceName, anchorName);
}
+ @Override
+ @Timed(value = "cps.data.service.datanode.delete.anchor.batch",
+ description = "Time taken to delete all datanodes for multiple anchors")
+ public void deleteDataNodes(final String dataspaceName, final Collection<String> anchorNames,
+ final OffsetDateTime observedTimestamp) {
+ cpsValidator.validateNameCharacters(dataspaceName);
+ cpsValidator.validateNameCharacters(anchorNames);
+ for (final String anchorName : anchorNames) {
+ processDataUpdatedEventAsync(dataspaceName, anchorName, ROOT_NODE_XPATH, DELETE, observedTimestamp);
+ }
+ cpsDataPersistenceService.deleteDataNodes(dataspaceName, anchorNames);
+ }
+
@Override
@Timed(value = "cps.data.service.list.delete",
description = "Time taken to delete a list or list element")
import io.micrometer.core.annotation.Timed;
import java.util.Collection;
import java.util.Map;
+import java.util.stream.Collectors;
import lombok.RequiredArgsConstructor;
import org.onap.cps.api.CpsAdminService;
import org.onap.cps.api.CpsModuleService;
public void deleteSchemaSetsWithCascade(final String dataspaceName, final Collection<String> schemaSetNames) {
cpsValidator.validateNameCharacters(dataspaceName);
cpsValidator.validateNameCharacters(schemaSetNames);
- for (final String schemaSetName : schemaSetNames) {
- final Collection<Anchor> anchors = cpsAdminService.getAnchors(dataspaceName, schemaSetName);
- for (final Anchor anchor : anchors) {
- cpsAdminService.deleteAnchor(dataspaceName, anchor.getName());
- }
- }
+ final Collection<String> anchorNames = cpsAdminService.getAnchors(dataspaceName, schemaSetNames)
+ .stream().map(Anchor::getName).collect(Collectors.toSet());
+ cpsAdminService.deleteAnchors(dataspaceName, anchorNames);
cpsModulePersistenceService.deleteUnusedYangResourceModules();
cpsModulePersistenceService.deleteSchemaSets(dataspaceName, schemaSetNames);
for (final String schemaSetName : schemaSetNames) {
void createAnchor(String dataspaceName, String schemaSetName, String anchorName);
/**
- * Read all anchors associated the given schema-set in the given dataspace.
+ * Read all anchors associated with the given schema-set in the given dataspace.
*
* @param dataspaceName dataspace name
* @param schemaSetName schema-set name
*/
Collection<Anchor> getAnchors(String dataspaceName, String schemaSetName);
+ /**
+ * Read all anchors associated with multiple schema-sets in the given dataspace.
+ *
+ * @param dataspaceName dataspace name
+ * @param schemaSetNames schema-set names
+ * @return a collection of anchors
+ */
+ Collection<Anchor> getAnchors(String dataspaceName, Collection<String> schemaSetNames);
+
/**
* Read all anchors in the given a dataspace.
*
* @param anchorName anchor name
*/
void deleteAnchor(String dataspaceName, String anchorName);
+
+ /**
+ * Delete anchors by name in given dataspace.
+ *
+ * @param dataspaceName dataspace name
+ * @param anchorNames anchor names
+ */
+ void deleteAnchors(String dataspaceName, Collection<String> anchorNames);
}
*/
void deleteDataNodes(String dataspaceName, String anchorName);
+ /**
+ * Deletes all dataNodes in multiple anchors.
+ *
+ * @param dataspaceName dataspace name
+ * @param anchorNames anchor names
+ */
+ void deleteDataNodes(String dataspaceName, Collection<String> anchorNames);
+
/**
* Deletes a single existing list element or the whole list.
*
/*
* ============LICENSE_START=======================================================
- * Copyright (C) 2020-2022 Nordix Foundation
+ * Copyright (C) 2020-2023 Nordix Foundation
* Modifications Copyright (C) 2020-2022 Bell Canada.
* Modifications Copyright (C) 2021 Pantheon.tech
* Modifications Copyright (C) 2022 TechMahindra Ltd.
1 * mockCpsValidator.validateNameCharacters('someDataspace', 'someSchemaSet')
}
+ def 'Retrieve all anchors for multiple schema-sets.'() {
+ given: 'that anchor is associated with the dataspace and schemasets'
+ def anchors = [new Anchor(), new Anchor()]
+ mockCpsAdminPersistenceService.getAnchors('someDataspace', _ as Collection<String>) >> anchors
+ when: 'get anchors is called for a dataspace name and schema set names'
+ def result = objectUnderTest.getAnchors('someDataspace', ['schemaSet1', 'schemaSet2'])
+ then: 'the collection provided by persistence service is returned as result'
+ result == anchors
+ and: 'the CpsValidator is called on the dataspace name and schema-set names'
+ 1 * mockCpsValidator.validateNameCharacters('someDataspace')
+ 1 * mockCpsValidator.validateNameCharacters(_)
+ }
+
def 'Retrieve anchor for dataspace and provided anchor name.'() {
given: 'that anchor name is associated with the dataspace'
Anchor anchor = new Anchor()
1 * mockCpsValidator.validateNameCharacters('someDataspace', 'someAnchor')
}
+ def 'Delete multiple anchors.'() {
+ when: 'delete anchors is invoked'
+ objectUnderTest.deleteAnchors('someDataspace', ['anchor1', 'anchor2'])
+ then: 'delete data nodes is invoked on the data service with expected parameters'
+ 1 * mockCpsDataService.deleteDataNodes('someDataspace', _ as Collection<String>, _ as OffsetDateTime)
+ and: 'the persistence service method is invoked with same parameters to delete anchor'
+ 1 * mockCpsAdminPersistenceService.deleteAnchors('someDataspace',_ as Collection<String>)
+ and: 'the CpsValidator is called on the dataspace name and anchor names'
+ 1 * mockCpsValidator.validateNameCharacters('someDataspace')
+ 1 * mockCpsValidator.validateNameCharacters(_)
+ }
+
def 'Query all anchor identifiers for a dataspace and module names.'() {
given: 'the persistence service is invoked with the expected parameters and returns a list of anchors'
mockCpsAdminPersistenceService.queryAnchors('some-dataspace-name', ['some-module-name']) >> [new Anchor(name:'some-anchor-identifier')]
1 * mockCpsDataPersistenceService.deleteDataNodes(dataspaceName, anchorName)
}
+ def 'Delete all data nodes for given dataspace and multiple anchors.'() {
+ given: 'schema set for given anchors and dataspace references test tree model'
+ setupSchemaSetMocks('test-tree.yang')
+ when: 'delete data node method is invoked with correct parameters'
+ objectUnderTest.deleteDataNodes(dataspaceName, ['anchor1', 'anchor2'], observedTimestamp)
+ then: 'data updated events are sent to notification service before the delete'
+ 2 * mockNotificationService.processDataUpdatedEvent(dataspaceName, _, '/', Operation.DELETE, observedTimestamp)
+ and: 'the CpsValidator is called on the dataspace name and the anchor names'
+ 2 * mockCpsValidator.validateNameCharacters(_)
+ and: 'the persistence service method is invoked with the correct parameters'
+ 1 * mockCpsDataPersistenceService.deleteDataNodes(dataspaceName, _ as Collection<String>)
+ }
+
def setupSchemaSetMocks(String... yangResources) {
def mockYangTextSchemaSourceSet = Mock(YangTextSchemaSourceSet)
mockYangTextSchemaSourceSetCache.get(dataspaceName, schemaSetName) >> mockYangTextSchemaSourceSet
def 'Delete multiple schema-sets when cascade is allowed.'() {
given: '#numberOfAnchors anchors are associated with each schemaset'
- mockCpsAdminService.getAnchors('my-dataspace', 'my-schemaset1') >> createAnchors(numberOfAnchors)
- mockCpsAdminService.getAnchors('my-dataspace', 'my-schemaset2') >> createAnchors(numberOfAnchors)
+ mockCpsAdminService.getAnchors('my-dataspace', ['my-schemaset1', 'my-schemaset2']) >> createAnchors(numberOfAnchors * 2)
when: 'schema set deletion is requested with cascade allowed'
objectUnderTest.deleteSchemaSetsWithCascade('my-dataspace', ['my-schemaset1', 'my-schemaset2'])
- then: 'anchor deletion is called 2 * #numberOfAnchors times'
- (2 * numberOfAnchors) * mockCpsAdminService.deleteAnchor('my-dataspace', _)
+ then: 'anchor deletion is called #numberOfAnchors times'
+ mockCpsAdminService.deleteAnchors('my-dataspace', _)
and: 'persistence service method is invoked with same parameters'
mockCpsModulePersistenceService.deleteSchemaSets('my-dataspace', _)
and: 'schema sets will be removed from the cache'
Code Review / cps.git / commitdiff