From: emaclee Date: Tue, 14 Oct 2025 12:56:38 +0000 (+0100) Subject: (Bug) Ensure clean up process also removes orphaned data in fragmenttable X-Git-Tag: 3.7.2~2^2~1 X-Git-Url: https://gerrit.onap.org/r/gitweb?a=commitdiff_plain;h=15e20fdf691556fd1c6a8d581dbd06716c02b06d;p=cps.git (Bug) Ensure clean up process also removes orphaned data in fragmenttable - Add method to remove orphaned data from fragment table to the endpoint /cps/api/{}/admin/dataspaces/{}/actions/clean - the condition for removing fragment table data is based on the following: - (a) if entity's xpath contains more than 1 '/' and the parent is null - (b) the parent id is equal to the id of an entity that meets condition (a) Issue-ID: CPS-3006 Change-Id: I1a7de57993c58c3b607b3fdee7476a29bdea5bc2 Signed-off-by: emaclee --- diff --git a/cps-rest/src/main/java/org/onap/cps/rest/controller/AdminRestController.java b/cps-rest/src/main/java/org/onap/cps/rest/controller/AdminRestController.java index ca7d374949..8bbd9d5ca5 100755 --- a/cps-rest/src/main/java/org/onap/cps/rest/controller/AdminRestController.java +++ b/cps-rest/src/main/java/org/onap/cps/rest/controller/AdminRestController.java @@ -192,6 +192,7 @@ public class AdminRestController implements CpsAdminApi { @Override public ResponseEntity cleanDataspace(final String apiVersion, final String dataspaceName) { cpsModuleService.deleteAllUnusedYangModuleData(dataspaceName); + cpsDataspaceService.deleteAllOrphanedData(dataspaceName); return new ResponseEntity<>(HttpStatus.NO_CONTENT); } diff --git a/cps-ri/src/main/java/org/onap/cps/ri/CpsAdminPersistenceServiceImpl.java b/cps-ri/src/main/java/org/onap/cps/ri/CpsAdminPersistenceServiceImpl.java index 27dd48d38c..ccd2fca961 100755 --- a/cps-ri/src/main/java/org/onap/cps/ri/CpsAdminPersistenceServiceImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/CpsAdminPersistenceServiceImpl.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2020-2025 Nordix Foundation. + * Copyright (C) 2020-2025 OpenInfra Foundation Europe. All rights reserved. * Modifications Copyright (C) 2020-2022 Bell Canada. * Modifications Copyright (C) 2021 Pantheon.tech * Modifications Copyright (C) 2022 TechMahindra Ltd. @@ -25,6 +25,9 @@ package org.onap.cps.ri; import jakarta.transaction.Transactional; import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -37,6 +40,7 @@ import org.onap.cps.ri.models.DataspaceEntity; import org.onap.cps.ri.models.SchemaSetEntity; import org.onap.cps.ri.repository.AnchorRepository; import org.onap.cps.ri.repository.DataspaceRepository; +import org.onap.cps.ri.repository.FragmentRepository; import org.onap.cps.ri.repository.SchemaSetRepository; import org.onap.cps.spi.CpsAdminPersistenceService; import org.springframework.dao.DataIntegrityViolationException; @@ -50,6 +54,7 @@ public class CpsAdminPersistenceServiceImpl implements CpsAdminPersistenceServic private final DataspaceRepository dataspaceRepository; private final AnchorRepository anchorRepository; private final SchemaSetRepository schemaSetRepository; + private final FragmentRepository fragmentRepository; @Override public void createDataspace(final String dataspaceName) { @@ -177,6 +182,17 @@ public class CpsAdminPersistenceServiceImpl implements CpsAdminPersistenceServic anchorRepository.updateAnchorSchemaSetId(schemaSetEntity.getId(), anchorEntity.getId()); } + @Override + public void deleteAllOrphanedFragmentEntities(final String dataspaceName) { + final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName); + final List anchorEntities = anchorRepository.getAnchorEntitiesByDataspace(dataspaceEntity); + final Set anchorIds = new HashSet<>(); + for (final AnchorEntity anchorEntity : anchorEntities) { + anchorIds.add(anchorEntity.getId()); + } + fragmentRepository.deleteOrphanedFragmentEntities(anchorIds); + } + private AnchorEntity getAnchorEntity(final String dataspaceName, final String anchorName) { final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName); return anchorRepository.getByDataspaceAndName(dataspaceEntity, anchorName); diff --git a/cps-ri/src/main/java/org/onap/cps/ri/repository/AnchorRepository.java b/cps-ri/src/main/java/org/onap/cps/ri/repository/AnchorRepository.java index aa6367dff2..b3e8b3a805 100755 --- a/cps-ri/src/main/java/org/onap/cps/ri/repository/AnchorRepository.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/repository/AnchorRepository.java @@ -1,7 +1,7 @@ /* * ============LICENSE_START======================================================= * Copyright (C) 2021 Pantheon.tech - * Modifications Copyright (C) 2021-2024 Nordix Foundation + * Modifications Copyright (C) 2021-2025 OpenInfra Foundation Europe. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ package org.onap.cps.ri.repository; import java.util.Collection; +import java.util.List; import java.util.Optional; import org.onap.cps.api.exceptions.AnchorNotFoundException; import org.onap.cps.ri.models.AnchorEntity; @@ -108,4 +109,5 @@ public interface AnchorRepository extends JpaRepository { @Query(value = "UPDATE anchor SET schema_set_id =:schemaSetId WHERE id = :anchorId ", nativeQuery = true) void updateAnchorSchemaSetId(@Param("schemaSetId") int schemaSetId, @Param("anchorId") long anchorId); + List getAnchorEntitiesByDataspace(DataspaceEntity dataspaceEntity); } diff --git a/cps-ri/src/main/java/org/onap/cps/ri/repository/FragmentRepository.java b/cps-ri/src/main/java/org/onap/cps/ri/repository/FragmentRepository.java index d95d322d37..c9957430db 100755 --- a/cps-ri/src/main/java/org/onap/cps/ri/repository/FragmentRepository.java +++ b/cps-ri/src/main/java/org/onap/cps/ri/repository/FragmentRepository.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2021-2024 Nordix Foundation. + * Copyright (C) 2021-2025 OpenInfra Foundation Europe. All rights reserved. * Modifications Copyright (C) 2020-2021 Bell Canada. * Modifications Copyright (C) 2020-2021 Pantheon.tech. * Modifications Copyright (C) 2023 TechMahindra Ltd. @@ -23,8 +23,10 @@ package org.onap.cps.ri.repository; +import jakarta.transaction.Transactional; import java.util.Collection; import java.util.List; +import java.util.Set; import org.onap.cps.ri.models.AnchorEntity; import org.onap.cps.ri.models.FragmentEntity; import org.onap.cps.ri.utils.EscapeUtils; @@ -106,4 +108,14 @@ public interface FragmentRepository extends JpaRepository, @Query(value = "SELECT * FROM fragment WHERE anchor_id = :anchorId AND parent_id IS NULL", nativeQuery = true) List findRootsByAnchorId(@Param("anchorId") long anchorId); + @Modifying + @Transactional + @Query(value = "DELETE FROM fragment where anchor_id IN (:anchorIds) " + + "AND (" + + "parent_id in" + + " (SELECT id from fragment WHERE (LENGTH(xpath) - LENGTH(REPLACE(xpath, '/','')) > 1)" + + " AND parent_id is null)" + + " OR ((LENGTH(xpath) - LENGTH(REPLACE(xpath, '/', '')) > 1) AND parent_id is null)" + + ")", nativeQuery = true) + void deleteOrphanedFragmentEntities(@Param("anchorIds") Set anchorIds); } diff --git a/cps-service/src/main/java/org/onap/cps/api/CpsDataspaceService.java b/cps-service/src/main/java/org/onap/cps/api/CpsDataspaceService.java index 32d57d44a6..f493eb73b0 100644 --- a/cps-service/src/main/java/org/onap/cps/api/CpsDataspaceService.java +++ b/cps-service/src/main/java/org/onap/cps/api/CpsDataspaceService.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2020-2023 Nordix Foundation + * Copyright (C) 2020-2025 OpenInfra Foundation Europe. All rights reserved. * Modifications Copyright (C) 2020-2022 Bell Canada. * Modifications Copyright (C) 2021 Pantheon.tech * Modifications Copyright (C) 2022 TechMahindra Ltd. @@ -63,4 +63,11 @@ public interface CpsDataspaceService { */ Collection getAllDataspaces(); + /** + * Delete orphaned data for a given dataspace name. + * + * @param dataspaceName the name of the dataspace where the data is located. + */ + void deleteAllOrphanedData(String dataspaceName); + } diff --git a/cps-service/src/main/java/org/onap/cps/impl/CpsDataspaceServiceImpl.java b/cps-service/src/main/java/org/onap/cps/impl/CpsDataspaceServiceImpl.java index ac55b81bdc..0abc97b0b2 100644 --- a/cps-service/src/main/java/org/onap/cps/impl/CpsDataspaceServiceImpl.java +++ b/cps-service/src/main/java/org/onap/cps/impl/CpsDataspaceServiceImpl.java @@ -61,4 +61,9 @@ public class CpsDataspaceServiceImpl implements CpsDataspaceService { return cpsAdminPersistenceService.getAllDataspaces(); } + @Override + public void deleteAllOrphanedData(final String dataspaceName) { + cpsValidator.validateNameCharacters(dataspaceName); + cpsAdminPersistenceService.deleteAllOrphanedFragmentEntities(dataspaceName); + } } diff --git a/cps-service/src/main/java/org/onap/cps/spi/CpsAdminPersistenceService.java b/cps-service/src/main/java/org/onap/cps/spi/CpsAdminPersistenceService.java index 62b26622de..b43bb5a145 100755 --- a/cps-service/src/main/java/org/onap/cps/spi/CpsAdminPersistenceService.java +++ b/cps-service/src/main/java/org/onap/cps/spi/CpsAdminPersistenceService.java @@ -151,4 +151,11 @@ public interface CpsAdminPersistenceService { * @param schemaSetName schema set name */ void updateAnchorSchemaSet(String dataspaceName, String anchorName, String schemaSetName); + + /** + * Delete all fragment entities that have no parent. + * + * @param dataspaceName dataspace name + */ + void deleteAllOrphanedFragmentEntities(String dataspaceName); } diff --git a/cps-service/src/test/groovy/org/onap/cps/impl/CpsDataspaceServiceImplSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/impl/CpsDataspaceServiceImplSpec.groovy index 97f6fba4d3..75b8948a80 100644 --- a/cps-service/src/test/groovy/org/onap/cps/impl/CpsDataspaceServiceImplSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/impl/CpsDataspaceServiceImplSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023 Nordix Foundation + * Copyright (C) 2023-2025 OpenInfra Foundation Europe. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -65,4 +65,11 @@ class CpsDataspaceServiceImplSpec extends Specification { 1 * mockCpsValidator.validateNameCharacters('someDataspace') } + def 'Delete all orphaned data.'(){ + when: 'deleting all orphaned data' + objectUnderTest.deleteAllOrphanedData('some-dataspaceName') + then: 'the persistence service method to delete all orphaned fragment entities is invoked' + 1 * mockCpsAdminPersistenceService.deleteAllOrphanedFragmentEntities('some-dataspaceName') + } + } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy index ba456ea904..b25f07c4ce 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/cps/DataspaceServiceIntegrationSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023-2025 Nordix Foundation + * Copyright (C) 2023-2025 OpenInfra Foundation Europe. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the 'License'); * you may not use this file except in compliance with the License. @@ -20,11 +20,18 @@ package org.onap.cps.integration.functional.cps +import static org.onap.cps.api.parameters.FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS +import static org.onap.cps.api.parameters.PaginationOption.NO_PAGINATION + import org.onap.cps.api.CpsDataspaceService +import java.time.OffsetDateTime +import org.onap.cps.api.exceptions.DataNodeNotFoundException import org.onap.cps.integration.base.FunctionalSpecBase import org.onap.cps.api.exceptions.AlreadyDefinedException import org.onap.cps.api.exceptions.DataspaceInUseException import org.onap.cps.api.exceptions.DataspaceNotFoundException +import org.onap.cps.ri.repository.FragmentRepository +import org.onap.cps.utils.ContentType class DataspaceServiceIntegrationSpec extends FunctionalSpecBase { @@ -104,4 +111,27 @@ class DataspaceServiceIntegrationSpec extends FunctionalSpecBase { thrown(AlreadyDefinedException) } + def 'Delete all orphaned data in a dataspace.'() { + setup: 'an anchor' + cpsAnchorService.createAnchor(GENERAL_TEST_DATASPACE, BOOKSTORE_SCHEMA_SET, 'testAnchor') + and: 'orphaned data' + def jsonDataMap = [:] + jsonDataMap.put('/bookstore/categories[@code=\'3\']', '{"books":[{"title": "Matilda"}]}') + jsonDataMap.put('/bookstore/categories[@code=\'3\']', '{"sub-categories":{"code":"1","additional-info":{"info-name":"sample"}}}') + cpsDataService.updateDataNodesAndDescendants(GENERAL_TEST_DATASPACE, 'testAnchor', jsonDataMap, OffsetDateTime.now(), ContentType.JSON) + def dataNodes = cpsDataService.getDataNodes(GENERAL_TEST_DATASPACE, 'testAnchor','/', INCLUDE_ALL_DESCENDANTS) + assert dataNodes.size() == 1 + assert dataNodes.childDataNodes.size() == 1 + and: 'parent node does not exist' + assert cpsQueryService.queryDataNodesAcrossAnchors(GENERAL_TEST_DATASPACE, '/bookstore', INCLUDE_ALL_DESCENDANTS, NO_PAGINATION).size() == 0 + when: 'deleting all orphaned data in a dataspace' + objectUnderTest.deleteAllOrphanedData(GENERAL_TEST_DATASPACE) + and: 'get data nodes in dataspace' + cpsDataService.getDataNodes(GENERAL_TEST_DATASPACE, 'testAnchor','/', INCLUDE_ALL_DESCENDANTS) + then: 'there will be no more data nodes available' + thrown(DataNodeNotFoundException) + cleanup: 'remove the data for this test' + cpsAnchorService.deleteAnchor(GENERAL_TEST_DATASPACE, 'testAnchor') + } + } diff --git a/integration-test/src/test/resources/data/bookstore/bookstore.yang b/integration-test/src/test/resources/data/bookstore/bookstore.yang index 0d093ea36c..9951decf31 100644 --- a/integration-test/src/test/resources/data/bookstore/bookstore.yang +++ b/integration-test/src/test/resources/data/bookstore/bookstore.yang @@ -102,6 +102,17 @@ module stores { type string; } + container sub-categories { + leaf code { + type string; + } + container additional-info { + leaf info-name { + type string; + } + } + } + list books { key title;