From: egernug Date: Wed, 25 Feb 2026 10:38:39 +0000 (+0000) Subject: Register new Datajob (Bulk) DMIs X-Git-Url: https://gerrit.onap.org/r/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F92%2F143392%2F10;p=cps.git Register new Datajob (Bulk) DMIs - Added new model dmi-registry:2026-01-28.yang - Added new DMI for datajob read and write in YangModelCmHandle - Added logic to delete old schema sets but keep the current and previous one Issue-ID: CPS-3138 Change-Id: I2873f81d398b105f7d624754ef52d0c15256a650 Signed-off-by: egernug --- diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/models/YangModelCmHandle.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/models/YangModelCmHandle.java index 39a8ff40f7..858f4696e4 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/models/YangModelCmHandle.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/models/YangModelCmHandle.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2021-2025 OpenInfra Foundation Europe. All rights reserved. + * Copyright (C) 2021-2026 OpenInfra Foundation Europe. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -63,6 +63,12 @@ public class YangModelCmHandle { @JsonProperty("dmi-model-service-name") private String dmiModelServiceName; + @JsonProperty("dmi-datajobs-read-service") + private String dmiDatajobsReadServiceName; + + @JsonProperty("dmi-datajobs-write-service") + private String dmiDatajobsWriteServiceName; + @JsonProperty("module-set-tag") private String moduleSetTag; diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/InventoryModelLoader.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/InventoryModelLoader.java index 491cc4160d..143e121673 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/InventoryModelLoader.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/InventoryModelLoader.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023-2025 OpenInfra Foundation Europe. All rights reserved. + * Copyright (C) 2023-2026 OpenInfra Foundation Europe. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,7 +41,8 @@ public class InventoryModelLoader extends AbstractModelLoader { private final ApplicationEventPublisher applicationEventPublisher; - private static final String CURRENT_SCHEMA_SET_NAME = "dmi-registry-2024-02-23"; + private static final String PREVIOUS_SCHEMA_SET_NAME = "dmi-registry-2024-02-23"; + private static final String CURRENT_SCHEMA_SET_NAME = "dmi-registry-2026-01-28"; private static final String INVENTORY_YANG_MODULE_NAME = "dmi-registry"; /** @@ -87,7 +88,7 @@ public class InventoryModelLoader extends AbstractModelLoader { private void installInventoryModel() { createDataspace(NCMP_DATASPACE_NAME); createDataspace(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME); - final String yangFileName = toYangFileName(CURRENT_SCHEMA_SET_NAME); + final String yangFileName = toYangFileName(); createSchemaSet(NCMP_DATASPACE_NAME, CURRENT_SCHEMA_SET_NAME, yangFileName); createAnchor(NCMP_DATASPACE_NAME, CURRENT_SCHEMA_SET_NAME, NCMP_DMI_REGISTRY_ANCHOR); createTopLevelDataNode(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR, INVENTORY_YANG_MODULE_NAME); @@ -96,26 +97,25 @@ public class InventoryModelLoader extends AbstractModelLoader { } private void deleteOldButNotThePreviousSchemaSets() { - //No schema sets passed in yet, but wil be required for future updates - deleteUnusedSchemaSets(NCMP_DATASPACE_NAME); - deleteUnusedSchemaSets(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME); + deleteUnusedSchemaSets(NCMP_DATASPACE_NAME, CURRENT_SCHEMA_SET_NAME, PREVIOUS_SCHEMA_SET_NAME); } private void upgradeInventoryModel() { - final String yangFileName = toYangFileName(CURRENT_SCHEMA_SET_NAME); + final String yangFileName = toYangFileName(); createSchemaSet(NCMP_DATASPACE_NAME, CURRENT_SCHEMA_SET_NAME, yangFileName); cpsAnchorService.updateAnchorSchemaSet(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR, CURRENT_SCHEMA_SET_NAME); log.info("Model Loader #2: Inventory upgraded successfully to model {}", CURRENT_SCHEMA_SET_NAME); + deleteOldButNotThePreviousSchemaSets(); } - private static String toYangFileName(final String schemaSetName) { - return INVENTORY_YANG_MODULE_NAME + "@" + getModuleRevision(schemaSetName) + ".yang"; + private static String toYangFileName() { + return INVENTORY_YANG_MODULE_NAME + "@" + getModuleRevision() + ".yang"; } - private static String getModuleRevision(final String schemaSetName) { + private static String getModuleRevision() { // Extract the revision part ( for example: 2024-02-23) - return schemaSetName.substring(INVENTORY_YANG_MODULE_NAME.length() + 1); + return CURRENT_SCHEMA_SET_NAME.substring(INVENTORY_YANG_MODULE_NAME.length() + 1); } } diff --git a/cps-ncmp-service/src/main/resources/models/dmi-registry@2026-01-28.yang b/cps-ncmp-service/src/main/resources/models/dmi-registry@2026-01-28.yang new file mode 100644 index 0000000000..ed2c3f87ae --- /dev/null +++ b/cps-ncmp-service/src/main/resources/models/dmi-registry@2026-01-28.yang @@ -0,0 +1,159 @@ +module dmi-registry { + + yang-version 1.1; + + namespace "org:onap:cps:ncmp"; + + prefix dmi-reg; + + contact "toine.siebelink@est.tech"; + + revision "2026-01-28" { + description + "Added dmi-datajobs-read-service, dmi-datajobs-write-service"; + } + + revision "2024-02-23" { + description + "Added data-producer-identifier"; + } + + revision "2023-11-27" { + description + "Added alternate-id"; + } + + revision "2023-08-23" { + description + "Added module-set-tag"; + } + + revision "2022-05-10" { + description + "Added data-sync-enabled, sync-state with state, last-sync-time, data-store-sync-state with operational and running sync-state"; + } + + revision "2022-02-10" { + description + "Added state, lock-reason, lock-reason-details to aid with cmHandle sync and timestamp to aid with retry/timeout scenarios"; + } + + revision "2021-12-13" { + description + "Added new list of public-properties and additional-properties for a Cm-Handle which are exposed to clients of the NCMP interface"; + } + + revision "2021-10-20" { + description + "Added dmi-data-service-name & dmi-model-service-name to allow separate DMI instances for each responsibility"; + } + + revision "2021-05-20" { + description + "Initial Version"; + } + + grouping LockReason { + leaf reason { + type string; + } + leaf details { + type string; + } + } + + grouping SyncState { + leaf sync-state { + type string; + } + leaf last-sync-time { + type string; + } + } + + grouping Datastores { + container operational { + uses SyncState; + } + container running { + uses SyncState; + } + } + + container dmi-registry { + list cm-handles { + key "id"; + leaf id { + type string; + } + leaf dmi-service-name { + type string; + } + leaf dmi-data-service-name { + type string; + } + leaf dmi-model-service-name { + type string; + } + leaf dmi-datajobs-read-service { + type string; + } + leaf dmi-datajobs-write-service { + type string; + } + leaf module-set-tag { + type string; + } + leaf alternate-id { + type string; + } + leaf data-producer-identifier { + type string; + } + + list additional-properties { + key "name"; + leaf name { + type string; + } + leaf value { + type string; + } + } + + list public-properties { + key "name"; + leaf name { + type string; + } + leaf value { + type string; + } + } + + container state { + leaf cm-handle-state { + type string; + } + + container lock-reason { + uses LockReason; + } + + leaf last-update-time { + type string; + } + + leaf data-sync-enabled { + type boolean; + default "false"; + } + + container datastores { + uses Datastores; + } + } + } + } +} + diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/InventoryModelLoaderSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/InventoryModelLoaderSpec.groovy index f8b30d4c2d..aa12dd5e3a 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/InventoryModelLoaderSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/InventoryModelLoaderSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023-2025 OpenInfra Foundation Europe. All rights reserved. + * Copyright (C) 2023-2026 OpenInfra Foundation Europe. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,12 +28,15 @@ import org.onap.cps.api.CpsDataService import org.onap.cps.api.CpsDataspaceService import org.onap.cps.api.CpsModuleService import org.onap.cps.api.exceptions.AnchorNotFoundException +import org.onap.cps.api.model.Anchor import org.onap.cps.api.model.Dataspace import org.onap.cps.api.model.ModuleDefinition +import org.onap.cps.api.model.SchemaSet import org.onap.cps.init.ModelLoaderLock import org.onap.cps.init.actuator.ReadinessManager import org.onap.cps.impl.CpsServicesBundle import org.slf4j.LoggerFactory +import org.springframework.boot.SpringApplication import org.springframework.boot.context.event.ApplicationReadyEvent import org.springframework.context.ApplicationEventPublisher import org.springframework.context.annotation.AnnotationConfigApplicationContext @@ -68,7 +71,7 @@ class InventoryModelLoaderSpec extends Specification { void setup() { objectUnderTest.isMaster = true - expectedNewYangResourceToContentMap = objectUnderTest.mapYangResourcesToContent('dmi-registry@2024-02-23.yang') + expectedNewYangResourceToContentMap = objectUnderTest.mapYangResourcesToContent('dmi-registry@2026-01-28.yang') logger.setLevel(Level.DEBUG) loggingListAppender = new ListAppender() logger.addAppender(loggingListAppender) @@ -86,12 +89,14 @@ class InventoryModelLoaderSpec extends Specification { mockCpsAdminService.getDataspace(NCMP_DATASPACE_NAME) >> new Dataspace('') and: 'module revision does not exist' mockCpsModuleService.getModuleDefinitionsByAnchorAndModule(_, _, _, _) >> Collections.emptyList() + and: 'no schema sets exist in the dataspace' + mockCpsModuleService.getSchemaSets(_) >> [] + and: 'anchor does not exist' + mockCpsAnchorService.getAnchor(_, _) >> { throw new AnchorNotFoundException('dataspace', 'anchor') } when: 'the application is ready' - objectUnderTest.onApplicationEvent(Mock(ApplicationReadyEvent)) + objectUnderTest.onApplicationEvent(new ApplicationReadyEvent(Mock(SpringApplication), null, applicationContext, null)) then: 'the module service is used to create the new schema set from the correct resource' - 1 * mockCpsModuleService.createSchemaSet(NCMP_DATASPACE_NAME, 'dmi-registry-2024-02-23', expectedNewYangResourceToContentMap) - and: 'No schema sets are being removed by the module service (yet)' - 0 * mockCpsModuleService.deleteSchemaSet(NCMP_DATASPACE_NAME, _, _) + 1 * mockCpsModuleService.createSchemaSet(NCMP_DATASPACE_NAME, 'dmi-registry-2026-01-28', expectedNewYangResourceToContentMap) and: 'application event publisher is called once' 1 * mockApplicationEventPublisher.publishEvent(_) } @@ -100,22 +105,28 @@ class InventoryModelLoaderSpec extends Specification { given: 'the anchor and module revision does not exist' mockCpsAnchorService.getAnchor(_, _) >> { throw new AnchorNotFoundException('', '') } mockCpsModuleService.getModuleDefinitionsByAnchorAndModule(_, _, _, _) >> Collections.emptyList() + and: 'no schema sets exist in the dataspace' + mockCpsModuleService.getSchemaSets(_) >> [] when: 'the inventory model loader is triggered' objectUnderTest.onboardOrUpgradeModel() then: 'a new schema set for the 2025-07-22 revision is installed' - 1 * mockCpsModuleService.createSchemaSet(NCMP_DATASPACE_NAME, 'dmi-registry-2024-02-23', expectedNewYangResourceToContentMap) + 1 * mockCpsModuleService.createSchemaSet(NCMP_DATASPACE_NAME, 'dmi-registry-2026-01-28', expectedNewYangResourceToContentMap) } def 'Upgrade model revision'() { given: 'the anchor exists and new module revision is not installed' + def mockAnchor = Mock(Anchor) { getSchemaSetName() >> 'dmi-registry-2024-02-23' } mockCpsAnchorService.getAnchor(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR) >> {} mockCpsModuleService.getModuleDefinitionsByAnchorAndModule(_, _, _, _) >> Collections.emptyList() + and: 'dataspace has the old schema set' + def oldSchemaSet = Mock(SchemaSet) { getName() >> 'dmi-registry-2024-02-23' } + mockCpsModuleService.getSchemaSets(_) >> [oldSchemaSet] when: 'the inventory model loader is triggered' objectUnderTest.onboardOrUpgradeModel() then: 'the new schema set for the 2025-07-22 revision is created' - 1 * mockCpsModuleService.createSchemaSet(NCMP_DATASPACE_NAME, 'dmi-registry-2024-02-23', expectedNewYangResourceToContentMap) + 1 * mockCpsModuleService.createSchemaSet(NCMP_DATASPACE_NAME, 'dmi-registry-2026-01-28', expectedNewYangResourceToContentMap) and: 'the anchor is updated to point to the new schema set' - 1 * mockCpsAnchorService.updateAnchorSchemaSet(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR, 'dmi-registry-2024-02-23') + 1 * mockCpsAnchorService.updateAnchorSchemaSet(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR, 'dmi-registry-2026-01-28') and: 'log messages confirm successful upgrade' assert loggingListAppender.list.any { it.message.contains("Inventory upgraded successfully") } } @@ -142,3 +153,4 @@ class InventoryModelLoaderSpec extends Specification { } } + diff --git a/cps-service/src/main/java/org/onap/cps/init/AbstractModelLoader.java b/cps-service/src/main/java/org/onap/cps/init/AbstractModelLoader.java index c0b45f43b2..4d377baa44 100644 --- a/cps-service/src/main/java/org/onap/cps/init/AbstractModelLoader.java +++ b/cps-service/src/main/java/org/onap/cps/init/AbstractModelLoader.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023-2025 OpenInfra Foundation Europe. All rights reserved. + * Copyright (C) 2023-2026 OpenInfra Foundation Europe. All rights reserved. * Modifications Copyright (C) 2024 Deutsche Telekom AG * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); @@ -40,6 +40,7 @@ import org.onap.cps.api.exceptions.DataspaceNotFoundException; import org.onap.cps.api.exceptions.DuplicatedYangResourceException; import org.onap.cps.api.exceptions.ModelOnboardingException; import org.onap.cps.api.model.ModuleDefinition; +import org.onap.cps.api.model.SchemaSet; import org.onap.cps.api.parameters.CascadeDeleteAllowed; import org.onap.cps.init.actuator.ReadinessManager; import org.onap.cps.utils.JsonObjectMapper; @@ -168,17 +169,26 @@ public abstract class AbstractModelLoader implements ModelLoader { } /** - * Delete unused schema set. + * Delete unused schema sets. * @param dataspaceName dataspace name - * @param schemaSetNames schema set names + * @param currentSchemaSetName current schema set name to keep + * @param previousSchemaSetName previous schema set name to keep (can be null) */ - public void deleteUnusedSchemaSets(final String dataspaceName, final String... schemaSetNames) { - for (final String schemaSetName : schemaSetNames) { - try { - cpsModuleService.deleteSchemaSet( - dataspaceName, schemaSetName, CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED); - } catch (final Exception exception) { - log.warn("Deleting schema set failed: {} ", exception.getMessage()); + public void deleteUnusedSchemaSets(final String dataspaceName, final String currentSchemaSetName, + final String previousSchemaSetName) { + final Collection allSchemaSets = cpsModuleService.getSchemaSets(dataspaceName); + for (final SchemaSet schemaSet : allSchemaSets) { + final String schemaSetName = schemaSet.getName(); + if (schemaSetName.startsWith("dmi-registry") + && !schemaSetName.equals(currentSchemaSetName) + && !schemaSetName.equals(previousSchemaSetName)) { + try { + log.info("Model Loader #2: Deleting schema {}", schemaSetName); + cpsModuleService.deleteSchemaSet( + dataspaceName, schemaSetName, CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED); + } catch (final Exception exception) { + log.warn("Deleting schema set {} failed: {}", schemaSetName, exception.getMessage()); + } } } } diff --git a/cps-service/src/test/groovy/org/onap/cps/init/AbstractModelLoaderSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/init/AbstractModelLoaderSpec.groovy index ee1c91778e..701c91d8b8 100644 --- a/cps-service/src/test/groovy/org/onap/cps/init/AbstractModelLoaderSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/init/AbstractModelLoaderSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2023-2025 OpenInfra Foundation Europe. All rights reserved. + * Copyright (C) 2023-2026 OpenInfra Foundation Europe. All rights reserved. * Modification Copyright (C) 2024 Deutsche Telekom AG * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); @@ -34,6 +34,7 @@ import org.onap.cps.api.exceptions.DataspaceNotFoundException import org.onap.cps.api.exceptions.DuplicatedYangResourceException import org.onap.cps.api.exceptions.ModelOnboardingException import org.onap.cps.api.model.ModuleDefinition +import org.onap.cps.api.model.SchemaSet import org.onap.cps.api.parameters.CascadeDeleteAllowed import org.onap.cps.init.actuator.ReadinessManager import org.slf4j.LoggerFactory @@ -210,24 +211,35 @@ class AbstractModelLoaderSpec extends Specification { } def 'Delete unused schema sets.'() { - when: 'unused schema sets get deleted' - objectUnderTest.deleteUnusedSchemaSets('some dataspace','schema set 1', 'schema set 2') - then: 'a request to delete each (without cascade) is delegated to the module service' - 1 * mockCpsModuleService.deleteSchemaSet('some dataspace', 'schema set 1', CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED) - 1 * mockCpsModuleService.deleteSchemaSet('some dataspace', 'schema set 2', CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED) + given: 'dataspace has 3 schema sets' + def schemaSet1 = Mock(SchemaSet) { getName() >> 'dmi-registry-2024-01-01' } + def schemaSet2 = Mock(SchemaSet) { getName() >> 'dmi-registry-2024-02-01' } + def schemaSet3 = Mock(SchemaSet) { getName() >> 'dmi-registry-2024-03-01' } + mockCpsModuleService.getSchemaSets('some dataspace') >> [schemaSet1, schemaSet2, schemaSet3] + when: 'deleting unused schema sets keeping schema set 2 and schema set 3' + objectUnderTest.deleteUnusedSchemaSets('some dataspace', 'dmi-registry-2024-02-01', 'dmi-registry-2024-03-01') + then: 'only schema set 1 is deleted' + 1 * mockCpsModuleService.deleteSchemaSet('some dataspace', 'dmi-registry-2024-01-01', CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED) + 0 * mockCpsModuleService.deleteSchemaSet('some dataspace', 'dmi-registry-2024-02-01', _) + 0 * mockCpsModuleService.deleteSchemaSet('some dataspace', 'dmi-registry-2024-03-01', _) } def 'Delete unused schema sets with exception.'() { - given: 'deleting the first schemaset causes an exception' - mockCpsModuleService.deleteSchemaSet(_, 'schema set 1', _) >> { throw new RuntimeException('test message')} - when: 'several unused schemas are deleted ' - objectUnderTest.deleteUnusedSchemaSets('some dataspace','schema set 1', 'schema set 2') + given: 'dataspace has 3 schema sets' + def schemaSet1 = Mock(SchemaSet) { getName() >> 'dmi-registry-2024-01-01' } + def schemaSet2 = Mock(SchemaSet) { getName() >> 'dmi-registry-2024-02-01' } + def schemaSet3 = Mock(SchemaSet) { getName() >> 'dmi-registry-2024-03-01' } + mockCpsModuleService.getSchemaSets('some dataspace') >> [schemaSet1, schemaSet2, schemaSet3] + and: 'deleting schema set 1 causes an exception' + mockCpsModuleService.deleteSchemaSet(_, 'dmi-registry-2024-01-01', _) >> { throw new RuntimeException('test message')} + when: 'deleting unused schemas keeping only schema set 2' + objectUnderTest.deleteUnusedSchemaSets('some dataspace', 'dmi-registry-2024-02-01', null) then: 'the exception message is logged' def logs = loggingListAppender.list.toString() - assert logs.contains('Deleting schema set failed') + assert logs.contains('Deleting schema set') assert logs.contains('test message') - and: 'the second schema set is still deleted' - 1 * mockCpsModuleService.deleteSchemaSet('some dataspace', 'schema set 2', CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED) + and: 'schema set 3 is still deleted' + 1 * mockCpsModuleService.deleteSchemaSet('some dataspace', 'dmi-registry-2024-03-01', CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED) } def 'Update anchor schema set.'() {