From: ToineSiebelink Date: Mon, 28 Aug 2023 14:21:59 +0000 (+0100) Subject: Extend model loader to support model-upgrade (part 2) X-Git-Tag: 3.3.7~12 X-Git-Url: https://gerrit.onap.org/r/gitweb?a=commitdiff_plain;h=c1183d7d14f1257b5dfa3dc67fa2f0814d1f4598;p=cps.git Extend model loader to support model-upgrade (part 2) - add upgrade related methods to common abstract class - add new (agreed) inventory model - add InventoryModelLoader - add more logging for success cases - simplified constant names considering the context (class name) Issue-ID: CPS-1804 Signed-off-by: ToineSiebelink Change-Id: I61a5c6d320d340a5c469ce20140f984439ba71a2 --- diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/AbstractModelLoader.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/AbstractModelLoader.java index 349b1c5b0..cb2e15a3c 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/AbstractModelLoader.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/AbstractModelLoader.java @@ -32,15 +32,14 @@ import org.onap.cps.api.CpsAdminService; import org.onap.cps.api.CpsDataService; import org.onap.cps.api.CpsModuleService; import org.onap.cps.ncmp.api.impl.exception.NcmpStartUpException; +import org.onap.cps.spi.CascadeDeleteAllowed; import org.onap.cps.spi.exceptions.AlreadyDefinedException; import org.onap.cps.utils.JsonObjectMapper; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.SpringApplication; import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.stereotype.Service; @Slf4j -@Service @RequiredArgsConstructor abstract class AbstractModelLoader implements ModelLoader { @@ -69,6 +68,7 @@ abstract class AbstractModelLoader implements ModelLoader { } void waitUntilDataspaceIsAvailable(final String dataspaceName) { + log.info("Model Loader start-up, waiting for database to be ready"); int attemptCount = 0; while (cpsAdminService.getDataspace(dataspaceName) == null) { if (attemptCount < maximumAttemptCount) { @@ -92,36 +92,54 @@ abstract class AbstractModelLoader implements ModelLoader { } catch (final AlreadyDefinedException alreadyDefinedException) { log.warn("Creating new schema set failed as schema set already exists"); } catch (final Exception exception) { - log.error("Creating schema set for subscription model failed: {} ", exception.getMessage()); + log.error("Creating schema set failed: {} ", exception.getMessage()); throw new NcmpStartUpException("Creating schema set failed", exception.getMessage()); } } + void deleteUnusedSchemaSets(final String dataspaceName, final String... schemaSetNames) { + for (final String schemaSetName : schemaSetNames) { + try { + cpsModuleService.deleteSchemaSet( + dataspaceName, schemaSetName, CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED); + } catch (final Exception exception) { + log.warn("Deleting schema set failed: {} ", exception.getMessage()); + } + } + } + void createAnchor(final String dataspaceName, final String schemaSetName, final String anchorName) { try { cpsAdminService.createAnchor(dataspaceName, schemaSetName, anchorName); } catch (final AlreadyDefinedException alreadyDefinedException) { log.warn("Creating new anchor failed as anchor already exists"); } catch (final Exception exception) { - log.error("Creating anchor for subscription model failed: {} ", exception.getMessage()); + log.error("Creating anchor failed: {} ", exception.getMessage()); throw new NcmpStartUpException("Creating anchor failed", exception.getMessage()); } } - void createTopLevelDataNode(final String dataspaceName, - final String anchorName, - final String dataNodeName) { + void createTopLevelDataNode(final String dataspaceName, final String anchorName, final String dataNodeName) { final String nodeData = jsonObjectMapper.asJsonString(Map.of(dataNodeName, Map.of())); try { cpsDataService.saveData(dataspaceName, anchorName, nodeData, OffsetDateTime.now()); } catch (final AlreadyDefinedException exception) { log.warn("Creating new data node '{}' failed as data node already exists", dataNodeName); } catch (final Exception exception) { - log.error("Creating data node for subscription model failed: {}", exception.getMessage()); + log.error("Creating data node failed: {}", exception.getMessage()); throw new NcmpStartUpException("Creating data node failed", exception.getMessage()); } } + void updateAnchorSchemaSet(final String dataspaceName, final String anchorName, final String schemaSetName) { + try { + cpsAdminService.updateAnchorSchemaSet(dataspaceName, anchorName, schemaSetName); + } catch (final Exception exception) { + log.error("Updating schema set failed: {}", exception.getMessage()); + throw new NcmpStartUpException("Updating schema set failed", exception.getMessage()); + } + } + Map createYangResourceToContentMap(final String resourceName) { return Map.of(resourceName, getFileContentAsString("models/" + resourceName)); } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/InventoryModelLoader.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/InventoryModelLoader.java new file mode 100644 index 000000000..5316d666d --- /dev/null +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/InventoryModelLoader.java @@ -0,0 +1,62 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.ncmp.init; + +import lombok.extern.slf4j.Slf4j; +import org.onap.cps.api.CpsAdminService; +import org.onap.cps.api.CpsDataService; +import org.onap.cps.api.CpsModuleService; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +public class InventoryModelLoader extends AbstractModelLoader { + + private static final String NEW_MODEL_FILE_NAME = "dmi-registry@2023-08-23.yang"; + private static final String NEW_SCHEMA_SET_NAME = "dmi-registry-2023-08-23"; + private static final String DATASPACE_NAME = "NCMP-Admin"; + private static final String ANCHOR_NAME = "ncmp-dmi-registry"; + + public InventoryModelLoader(final CpsAdminService cpsAdminService, + final CpsModuleService cpsModuleService, + final CpsDataService cpsDataService) { + super(cpsAdminService, cpsModuleService, cpsDataService); + } + + @Override + public void onboardOrUpgradeModel() { + waitUntilDataspaceIsAvailable(DATASPACE_NAME); + updateInventoryModel(); + log.info("Inventory Model updated successfully"); + } + + private void updateInventoryModel() { + createSchemaSet(DATASPACE_NAME, NEW_SCHEMA_SET_NAME, NEW_MODEL_FILE_NAME); + updateAnchorSchemaSet(DATASPACE_NAME, ANCHOR_NAME, NEW_SCHEMA_SET_NAME); + deleteOldButNotThePreviousSchemaSets(); + } + + private void deleteOldButNotThePreviousSchemaSets() { + //No schema sets passed in yet, but wil be required for future updates + deleteUnusedSchemaSets(DATASPACE_NAME); + } + +} diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/SubscriptionModelLoader.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/SubscriptionModelLoader.java index 614efd4f4..891244c7c 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/SubscriptionModelLoader.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/SubscriptionModelLoader.java @@ -31,11 +31,11 @@ import org.springframework.stereotype.Service; @Service public class SubscriptionModelLoader extends AbstractModelLoader { - private static final String SUBSCRIPTION_MODEL_FILENAME = "subscription.yang"; - private static final String SUBSCRIPTION_DATASPACE_NAME = "NCMP-Admin"; - private static final String SUBSCRIPTION_ANCHOR_NAME = "AVC-Subscriptions"; - private static final String SUBSCRIPTION_SCHEMASET_NAME = "subscriptions"; - private static final String SUBSCRIPTION_REGISTRY_DATANODE_NAME = "subscription-registry"; + private static final String MODEL_FILENAME = "subscription.yang"; + private static final String DATASPACE_NAME = "NCMP-Admin"; + private static final String ANCHOR_NAME = "AVC-Subscriptions"; + private static final String SCHEMASET_NAME = "subscriptions"; + private static final String REGISTRY_DATANODE_NAME = "subscription-registry"; public SubscriptionModelLoader(final CpsAdminService cpsAdminService, final CpsModuleService cpsModuleService, @@ -49,18 +49,18 @@ public class SubscriptionModelLoader extends AbstractModelLoader { @Override public void onboardOrUpgradeModel() { if (subscriptionModelLoaderEnabled) { - waitUntilDataspaceIsAvailable(SUBSCRIPTION_DATASPACE_NAME); + waitUntilDataspaceIsAvailable(DATASPACE_NAME); onboardSubscriptionModel(); + log.info("Subscription Model onboarded successfully"); } else { log.info("Subscription Model Loader is disabled"); } } private void onboardSubscriptionModel() { - createSchemaSet(SUBSCRIPTION_DATASPACE_NAME, SUBSCRIPTION_SCHEMASET_NAME, SUBSCRIPTION_MODEL_FILENAME); - createAnchor(SUBSCRIPTION_DATASPACE_NAME, SUBSCRIPTION_SCHEMASET_NAME, SUBSCRIPTION_ANCHOR_NAME); - createTopLevelDataNode(SUBSCRIPTION_DATASPACE_NAME, SUBSCRIPTION_ANCHOR_NAME, - SUBSCRIPTION_REGISTRY_DATANODE_NAME); + createSchemaSet(DATASPACE_NAME, SCHEMASET_NAME, MODEL_FILENAME); + createAnchor(DATASPACE_NAME, SCHEMASET_NAME, ANCHOR_NAME); + createTopLevelDataNode(DATASPACE_NAME, ANCHOR_NAME, REGISTRY_DATANODE_NAME); } } diff --git a/cps-ncmp-service/src/main/resources/models/dmi-registry@2023-08-23.yang b/cps-ncmp-service/src/main/resources/models/dmi-registry@2023-08-23.yang new file mode 100644 index 000000000..bb7604d91 --- /dev/null +++ b/cps-ncmp-service/src/main/resources/models/dmi-registry@2023-08-23.yang @@ -0,0 +1,131 @@ +module dmi-registry { + + yang-version 1.1; + + namespace "org:onap:cps:ncmp"; + + prefix dmi-reg; + + contact "toine.siebelink@est.tech"; + + revision "2023-08-23" { + description + "Added ModuleSetTag"; + } + + revision "2022-05-10" { + description + "Added DataSyncEnabled, SyncState with State, LastSyncTime, DataStoreSyncState with Operational and Running syncstate"; + } + + revision "2022-02-10" { + description + "Added State, LockReason, LockReasonDetails to aid with cmHandle sync and timestamp to aid with retry/timeout scenarios"; + } + + revision "2021-12-13" { + description + "Added new list of public additional properties for a Cm-Handle which are exposed to clients of the NCMP interface"; + } + + revision "2021-10-20" { + description + "Added dmi-data-service-name & dmi-model-service-name to allow separate DMI instances for each responsibility"; + } + + revision "2021-05-20" { + description + "Initial Version"; + } + + grouping LockReason { + leaf reason { + type string; + } + leaf details { + type string; + } + } + + grouping SyncState { + leaf sync-state { + type string; + } + leaf last-sync-time { + type string; + } + } + + grouping Datastores { + container operational { + uses SyncState; + } + container running { + uses SyncState; + } + } + + container dmi-registry { + list cm-handles { + key "id"; + leaf id { + type string; + } + leaf dmi-service-name { + type string; + } + leaf dmi-data-service-name { + type string; + } + leaf dmi-model-service-name { + type string; + } + leaf module-set-tag { + type string; + } + + list additional-properties { + key "name"; + leaf name { + type string; + } + leaf value { + type string; + } + } + + list public-properties { + key "name"; + leaf name { + type string; + } + leaf value { + type string; + } + } + + container state { + leaf cm-handle-state { + type string; + } + + container lock-reason { + uses LockReason; + } + + leaf last-update-time { + type string; + } + + leaf data-sync-enabled { + type boolean; + default "false"; + } + + container datastores { + uses Datastores; + } + } + } + } +} diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/AbstractModelLoaderSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/AbstractModelLoaderSpec.groovy index a271ca431..28eae8df1 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/AbstractModelLoaderSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/AbstractModelLoaderSpec.groovy @@ -27,6 +27,7 @@ import org.onap.cps.api.CpsAdminService import org.onap.cps.api.CpsDataService import org.onap.cps.api.CpsModuleService import org.onap.cps.ncmp.api.impl.exception.NcmpStartUpException +import org.onap.cps.spi.CascadeDeleteAllowed import org.onap.cps.spi.exceptions.AlreadyDefinedException import org.springframework.boot.SpringApplication import org.slf4j.LoggerFactory @@ -114,10 +115,32 @@ class AbstractModelLoaderSpec extends Specification { assert thrown.details.contains('unable to read file') } + def 'Delete unused schema sets.'() { + when: 'several unused schemas are deleted ' + objectUnderTest.deleteUnusedSchemaSets('some dataspace','schema set 1', 'schema set 2') + then: 'a request to delete each (without cascade) is delegated to the module service' + 1 * mockCpsModuleService.deleteSchemaSet('some dataspace', 'schema set 1', CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED) + 1 * mockCpsModuleService.deleteSchemaSet('some dataspace', 'schema set 2', CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED) + + } + + def 'Delete unused schema sets with exception.'() { + given: 'deleting the first schemaset causes an exception' + mockCpsModuleService.deleteSchemaSet(_, 'schema set 1', _) >> { throw new RuntimeException('test message')} + when: 'several unused schemas are deleted ' + objectUnderTest.deleteUnusedSchemaSets('some dataspace','schema set 1', 'schema set 2') + then: 'the exception message is logged' + def logs = loggingListAppender.list.toString() + assert logs.contains('Deleting schema set failed') + assert logs.contains('test message') + and: 'the second schema set is still deleted' + 1 * mockCpsModuleService.deleteSchemaSet('some dataspace', 'schema set 2', CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED) + } + def 'Create anchor.'() { when: 'creating an anchor' objectUnderTest.createAnchor('some dataspace','some schema set','new name') - then: 'thr operation is delegated to the admin service' + then: 'the operation is delegated to the admin service' 1 * mockCpsAdminService.createAnchor('some dataspace','some schema set', 'new name') } @@ -174,6 +197,24 @@ class AbstractModelLoaderSpec extends Specification { assert thrown.details.contains('test message') } + def 'Update anchor schema set.'() { + when: 'a schema set for an anchor is updated' + objectUnderTest.updateAnchorSchemaSet('some dataspace', 'anchor', 'new schema set') + then: 'the request is delegated to the admin service' + 1 * mockCpsAdminService.updateAnchorSchemaSet('some dataspace', 'anchor', 'new schema set') + } + + def 'Update anchor schema set with exception.'() { + given: 'the admin service throws an exception' + mockCpsAdminService.updateAnchorSchemaSet(*_) >> { throw new RuntimeException('test message') } + when: 'a schema set for an anchor is updated' + objectUnderTest.updateAnchorSchemaSet('some dataspace', 'anchor', 'new schema set') + then: 'a startup exception with correct message and details is thrown' + def thrown = thrown(NcmpStartUpException) + assert thrown.message.contains('Updating schema set failed') + assert thrown.details.contains('test message') + } + class TestModelLoader extends AbstractModelLoader { TestModelLoader(final CpsAdminService cpsAdminService, diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/InventoryModelLoaderSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/InventoryModelLoaderSpec.groovy new file mode 100644 index 000000000..9195bc74c --- /dev/null +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/InventoryModelLoaderSpec.groovy @@ -0,0 +1,75 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.ncmp.init + +import ch.qos.logback.classic.Level +import ch.qos.logback.classic.Logger +import ch.qos.logback.core.read.ListAppender +import org.onap.cps.api.CpsAdminService +import org.onap.cps.api.CpsDataService +import org.onap.cps.api.CpsModuleService +import org.onap.cps.spi.model.Dataspace +import org.slf4j.LoggerFactory +import org.springframework.boot.context.event.ApplicationReadyEvent +import org.springframework.context.annotation.AnnotationConfigApplicationContext +import spock.lang.Specification + +class InventoryModelLoaderSpec extends Specification { + + def mockCpsAdminService = Mock(CpsAdminService) + def mockCpsModuleService = Mock(CpsModuleService) + def mockCpsDataService = Mock(CpsDataService) + def objectUnderTest = new InventoryModelLoader(mockCpsAdminService, mockCpsModuleService, mockCpsDataService) + + def applicationContext = new AnnotationConfigApplicationContext() + + def expectedYangResourceToContentMap + def logger = (Logger) LoggerFactory.getLogger(objectUnderTest.class) + def loggingListAppender + + void setup() { + expectedYangResourceToContentMap = objectUnderTest.createYangResourceToContentMap('dmi-registry@2023-08-23.yang') + logger.setLevel(Level.DEBUG) + loggingListAppender = new ListAppender() + logger.addAppender(loggingListAppender) + loggingListAppender.start() + applicationContext.refresh() + } + + void cleanup() { + ((Logger) LoggerFactory.getLogger(SubscriptionModelLoader.class)).detachAndStopAllAppenders() + applicationContext.close() + } + + def 'Onboard subscription model via application ready event.'() { + given: 'dataspace is ready for use' + mockCpsAdminService.getDataspace('NCMP-Admin') >> new Dataspace('') + when: 'the application is ready' + objectUnderTest.onApplicationEvent(Mock(ApplicationReadyEvent)) + then: 'the module service is used to create the new schema set from the correct resource' + 1 * mockCpsModuleService.createSchemaSet('NCMP-Admin', 'dmi-registry-2023-08-23', expectedYangResourceToContentMap) + and: 'the admin service is used to update the anchor' + 1 * mockCpsAdminService.updateAnchorSchemaSet('NCMP-Admin', 'ncmp-dmi-registry', 'dmi-registry-2023-08-23') + and: 'No schema sets are being removed by the module service (yet)' + 0 * mockCpsModuleService.deleteSchemaSet('NCMP-Admin', _, _) + } + +} diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/SubscriptionModelLoaderSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/SubscriptionModelLoaderSpec.groovy index 305fe4c06..d99874ad6 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/SubscriptionModelLoaderSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/SubscriptionModelLoaderSpec.groovy @@ -41,12 +41,12 @@ class SubscriptionModelLoaderSpec extends Specification { def applicationContext = new AnnotationConfigApplicationContext() - def yangResourceToContentMap + def expectedYangResourceToContentMap def logger = (Logger) LoggerFactory.getLogger(objectUnderTest.class) def loggingListAppender void setup() { - yangResourceToContentMap = objectUnderTest.createYangResourceToContentMap('subscription.yang') + expectedYangResourceToContentMap = objectUnderTest.createYangResourceToContentMap('subscription.yang') logger.setLevel(Level.DEBUG) loggingListAppender = new ListAppender() logger.addAppender(loggingListAppender) @@ -67,7 +67,7 @@ class SubscriptionModelLoaderSpec extends Specification { when: 'the application is ready' objectUnderTest.onApplicationEvent(Mock(ApplicationReadyEvent)) then: 'the module service to create schema set is called once' - 1 * mockCpsModuleService.createSchemaSet('NCMP-Admin', 'subscriptions', yangResourceToContentMap) + 1 * mockCpsModuleService.createSchemaSet('NCMP-Admin', 'subscriptions', expectedYangResourceToContentMap) and: 'the admin service to create an anchor set is called once' 1 * mockCpsAdminService.createAnchor('NCMP-Admin', 'subscriptions', 'AVC-Subscriptions') and: 'the data service to create a top level datanode is called once'