From: mpriyank Date: Thu, 2 Oct 2025 15:39:58 +0000 (+0100) Subject: Fix the application readiness state during migration X-Git-Tag: 3.7.2~7^2 X-Git-Url: https://gerrit.onap.org/r/gitweb?a=commitdiff_plain;h=0516e22ea09f77155621469c95a487f56f94d2de;p=cps.git Fix the application readiness state during migration - updating the readiness health indicator to just work on the readiness state and not on the overall state. - before this fix, during migration the readiness, liveness and overall status is DOWN. - after the fix, readiness and overall state would be DOWN, while the application will be live, hence liveness probe will show as UP - ReadinessManager relies on ApplicationReadyEvent now instead of ApplicationStartedEvent - Dynamically register the model loaders during Application startup - removed the initial delay string from the watchdogs - Added a new arch rule for the cps init package to be used in ncmp-service - removed the module-sync-delayed package Issue-ID: CPS-2993 Change-Id: I4c3bbc8633b84984095ad11ca8e4fbade91fd0f2 Signed-off-by: mpriyank --- diff --git a/cps-application/src/main/resources/application.yml b/cps-application/src/main/resources/application.yml index d735a1c556..90e5ac1a42 100644 --- a/cps-application/src/main/resources/application.yml +++ b/cps-application/src/main/resources/application.yml @@ -229,10 +229,8 @@ ncmp: timers: advised-modules-sync: - initial-delay-ms: 40000 sleep-time-ms: 5000 cm-handle-data-sync: - initial-delay-ms: 40000 sleep-time-ms: 30000 model-loader: retry-time-ms: 1000 diff --git a/cps-application/src/test/java/org/onap/cps/architecture/NcmpArchitectureTest.java b/cps-application/src/test/java/org/onap/cps/architecture/NcmpArchitectureTest.java index 7f4e63f043..342211755e 100644 --- a/cps-application/src/test/java/org/onap/cps/architecture/NcmpArchitectureTest.java +++ b/cps-application/src/test/java/org/onap/cps/architecture/NcmpArchitectureTest.java @@ -77,6 +77,7 @@ public class NcmpArchitectureTest extends ArchitectureTestBase { "org.onap.cps.cpspath..", "org.onap.cps.events..", "org.onap.cps.impl..", - "org.onap.cps.utils..")); + "org.onap.cps.utils..", + "org.onap.cps.init..")); } diff --git a/cps-application/src/test/resources/application.yml b/cps-application/src/test/resources/application.yml index b08a3759ff..0bb43b0bb6 100644 --- a/cps-application/src/test/resources/application.yml +++ b/cps-application/src/test/resources/application.yml @@ -226,10 +226,8 @@ ncmp: timers: advised-modules-sync: - initial-delay-ms: 40000 sleep-time-ms: 5000 cm-handle-data-sync: - initial-delay-ms: 40000 sleep-time-ms: 30000 model-loader: retry-time-ms: 1000 diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/DataSyncWatchdog.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/DataSyncWatchdog.java index 5e498bb3c6..7f0e38a632 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/DataSyncWatchdog.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/DataSyncWatchdog.java @@ -32,6 +32,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.onap.cps.api.CpsDataService; import org.onap.cps.api.CpsModuleService; +import org.onap.cps.init.actuator.ReadinessManager; import org.onap.cps.ncmp.api.inventory.DataStoreSyncState; import org.onap.cps.ncmp.api.inventory.models.CompositeState; import org.onap.cps.ncmp.impl.inventory.InventoryPersistence; @@ -52,13 +53,25 @@ public class DataSyncWatchdog { private final CpsDataService cpsDataService; private final ModuleOperationsUtils moduleOperationsUtils; private final IMap dataSyncSemaphores; + private final ReadinessManager readinessManager; /** * Execute Cm Handle poll which queries the cm handle state in 'READY' and Operational Datastore Sync State in + * 'UNSYNCHRONIZED' when the system is in ready state. + */ + @Scheduled(fixedDelayString = "${ncmp.timers.cm-handle-data-sync.sleep-time-ms:30000}") + public void scheduledUnsynchronizedReadyCmHandleForInitialDataSync() { + if (!readinessManager.isReady()) { + log.info("System is not ready yet"); + return; + } + executeUnsynchronizedReadyCmHandleForInitialDataSync(); + } + + /** + * This method queries the cm handle state in 'READY' and Operational Datastore Sync State in * 'UNSYNCHRONIZED'. */ - @Scheduled(initialDelayString = "${ncmp.timers.cm-handle-data-sync.initial-delay-ms:40000}", - fixedDelayString = "${ncmp.timers.cm-handle-data-sync.sleep-time-ms:30000}") public void executeUnsynchronizedReadyCmHandleForInitialDataSync() { final List unsynchronizedReadyCmHandles = moduleOperationsUtils.getUnsynchronizedReadyCmHandles(); diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdog.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdog.java index 2d71f39339..49261f65b2 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdog.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdog.java @@ -29,6 +29,7 @@ import java.util.HashSet; import java.util.concurrent.BlockingQueue; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.onap.cps.init.actuator.ReadinessManager; import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.scheduling.annotation.Scheduled; @@ -45,18 +46,32 @@ public class ModuleSyncWatchdog { private final ModuleSyncTasks moduleSyncTasks; @Qualifier("cpsAndNcmpLock") private final IMap cpsAndNcmpLock; + private final ReadinessManager readinessManager; private static final int MODULE_SYNC_BATCH_SIZE = 300; private static final String VALUE_FOR_HAZELCAST_IN_PROGRESS_MAP = "Started"; + /** * Check DB for any cm handles in 'ADVISED' state. * Queue and create batches to process them asynchronously. * This method will only finish when there are no more 'ADVISED' cm handles in the DB. - * This method is triggered on a configurable interval (ncmp.timers.advised-modules-sync.sleep-time-ms) + * This method is triggered on a configurable interval (ncmp.timers.advised-modules-sync.sleep-time-ms) and when the + * system is in the ready state. + */ + @Scheduled(fixedDelayString = "${ncmp.timers.advised-modules-sync.sleep-time-ms:5000}") + public void scheduledModuleSyncAdvisedCmHandles() { + if (!readinessManager.isReady()) { + log.info("System is not ready yet"); + return; + } + moduleSyncAdvisedCmHandles(); + } + + /** + * This method is used when we dont want the scheduled behaviour. + * Mainly used in the integration testware. */ - @Scheduled(initialDelayString = "${ncmp.timers.advised-modules-sync.initial-delay-ms:40000}", - fixedDelayString = "${ncmp.timers.advised-modules-sync.sleep-time-ms:5000}") public void moduleSyncAdvisedCmHandles() { log.debug("Processing module sync watchdog waking up."); populateWorkQueueIfNeeded(); diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/DataSyncWatchdogSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/DataSyncWatchdogSpec.groovy index 03a302910d..ff6a14a3fe 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/DataSyncWatchdogSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/DataSyncWatchdogSpec.groovy @@ -20,14 +20,20 @@ package org.onap.cps.ncmp.impl.inventory.sync +import ch.qos.logback.classic.Level +import ch.qos.logback.classic.Logger +import ch.qos.logback.classic.spi.ILoggingEvent +import ch.qos.logback.core.read.ListAppender import com.hazelcast.map.IMap import org.onap.cps.api.CpsDataService import org.onap.cps.api.CpsModuleService +import org.onap.cps.init.actuator.ReadinessManager import org.onap.cps.ncmp.api.inventory.models.CompositeState import org.onap.cps.ncmp.api.inventory.DataStoreSyncState import org.onap.cps.ncmp.impl.inventory.InventoryPersistence import org.onap.cps.ncmp.api.inventory.models.CmHandleState import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle +import org.slf4j.LoggerFactory import spock.lang.Specification import static org.onap.cps.ncmp.impl.inventory.NcmpPersistence.NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME @@ -39,25 +45,53 @@ class DataSyncWatchdogSpec extends Specification { def mockCpsDataService = Mock(CpsDataService) def mockModuleOperationUtils = Mock(ModuleOperationsUtils) def mockDataSyncSemaphores = Mock(IMap) + def mockReadinessManager = Mock(ReadinessManager) def jsonString = '{"stores:bookstore":{"categories":[{"code":"01"}]}}' - def objectUnderTest = new DataSyncWatchdog(mockInventoryPersistence, mockCpsModuleService, mockCpsDataService, mockModuleOperationUtils, mockDataSyncSemaphores) + def objectUnderTest = new DataSyncWatchdog(mockInventoryPersistence, mockCpsModuleService, mockCpsDataService, mockModuleOperationUtils, mockDataSyncSemaphores, mockReadinessManager) def compositeState = getCompositeState() def yangModelCmHandle1 = createSampleYangModelCmHandle('cm-handle-1') def yangModelCmHandle2 = createSampleYangModelCmHandle('cm-handle-2') + def logAppender = Spy(ListAppender) + + void setup() { + def logger = LoggerFactory.getLogger(DataSyncWatchdog) + logger.setLevel(Level.INFO) + logger.addAppender(logAppender) + logAppender.start() + } + + void cleanup() { + ((Logger) LoggerFactory.getLogger(DataSyncWatchdog.class)).detachAndStopAllAppenders() + } + + def 'Data sync watchdog is triggered'(){ + given: 'the system is not ready to accept traffic' + mockReadinessManager.isReady() >> false + when: 'data sync is started' + objectUnderTest.scheduledUnsynchronizedReadyCmHandleForInitialDataSync() + then: 'an event is logged with level INFO' + def loggingEvent = getLoggingEvent() + assert loggingEvent.level == Level.INFO + and: 'the log indicates that the system is not ready yet' + assert loggingEvent.formattedMessage == 'System is not ready yet' + } + def 'Data Sync for Cm Handle State in READY and Operational Sync State in UNSYNCHRONIZED.'() { given: 'sample resource data' def resourceData = jsonString + and: 'system is ready to accept traffic' + mockReadinessManager.isReady() >> true and: 'sync utilities returns a cm handle twice' mockModuleOperationUtils.getUnsynchronizedReadyCmHandles() >> [yangModelCmHandle1, yangModelCmHandle2] and: 'we have the module and root nodes references to form the options field' mockCpsModuleService.getRootNodeReferences(_, 'cm-handle-1') >> ['some-module-1:some-root-node'] mockCpsModuleService.getRootNodeReferences(_, 'cm-handle-2') >> ['some-module-2:some-root-node'] when: 'data sync poll is executed' - objectUnderTest.executeUnsynchronizedReadyCmHandleForInitialDataSync() + objectUnderTest.scheduledUnsynchronizedReadyCmHandleForInitialDataSync() then: 'the inventory persistence cm handle returns a composite state for the first cm handle' 1 * mockInventoryPersistence.getCmHandleState('cm-handle-1') >> compositeState and: 'the sync util returns first resource data' @@ -77,12 +111,14 @@ class DataSyncWatchdogSpec extends Specification { } def 'Data Sync for Cm Handle State in READY and Operational Sync State in UNSYNCHRONIZED without resource data.'() { - given: 'sync utilities returns a cm handle' + given: 'system is ready to accept traffic' + mockReadinessManager.isReady() >> true + and: 'sync utilities returns a cm handle' mockModuleOperationUtils.getUnsynchronizedReadyCmHandles() >> [yangModelCmHandle1] and: 'the module service returns the module and root nodes references to form the options field' mockCpsModuleService.getRootNodeReferences(_,'cm-handle-1') >> ['some-module-1:some-root-node'] when: 'data sync poll is executed' - objectUnderTest.executeUnsynchronizedReadyCmHandleForInitialDataSync() + objectUnderTest.scheduledUnsynchronizedReadyCmHandleForInitialDataSync() then: 'the inventory persistence cm handle returns a composite state for the first cm handle' 1 * mockInventoryPersistence.getCmHandleState('cm-handle-1') >> compositeState and: 'the sync util returns no resource data' @@ -92,40 +128,46 @@ class DataSyncWatchdogSpec extends Specification { } def 'Data Sync for Cm Handle that is already being processed.'() { - given: 'sync utilities returns a cm handle' + given: 'system is ready to accept traffic' + mockReadinessManager.isReady() >> true + and: 'sync utilities returns a cm handle' mockModuleOperationUtils.getUnsynchronizedReadyCmHandles() >> [yangModelCmHandle1] and: 'the module service returns the module and root nodes references to form the options field' mockCpsModuleService.getRootNodeReferences(_,'cm-handle-1') >> ['some-module-1:some-root-node'] and: 'the shared data sync semaphore indicate it is already being processed' mockDataSyncSemaphores.putIfAbsent('cm-handle-1', _, _, _) >> 'something (not null)' when: 'data sync poll is executed' - objectUnderTest.executeUnsynchronizedReadyCmHandleForInitialDataSync() + objectUnderTest.scheduledUnsynchronizedReadyCmHandleForInitialDataSync() then: 'it is NOT processed e.g. state is not requested' 0 * mockInventoryPersistence.getCmHandleState(*_) } def 'Data sync handles exception during overall cm handle processing.'() { - given: 'sync utilities returns a cm handle' + given: 'system is ready to accept traffic' + mockReadinessManager.isReady() >> true + and: 'sync utilities returns a cm handle' mockModuleOperationUtils.getUnsynchronizedReadyCmHandles() >> [yangModelCmHandle1] and: 'semaphore map allows processing' mockDataSyncSemaphores.putIfAbsent('cm-handle-1', false, _, _) >> null and: 'getting cm handle state throws exception' mockInventoryPersistence.getCmHandleState('cm-handle-1') >> { throw new RuntimeException('some exception') } when: 'data sync poll is executed' - objectUnderTest.executeUnsynchronizedReadyCmHandleForInitialDataSync() + objectUnderTest.scheduledUnsynchronizedReadyCmHandleForInitialDataSync() then: 'no exception is thrown' noExceptionThrown() } def 'Data sync handles exception during resource data retrieval.'() { - given: 'sync utilities returns a cm handle' + given: 'system is ready to accept traffic' + mockReadinessManager.isReady() >> true + and: 'sync utilities returns a cm handle' mockModuleOperationUtils.getUnsynchronizedReadyCmHandles() >> [yangModelCmHandle1] and: 'semaphore map allows processing' mockDataSyncSemaphores.putIfAbsent('cm-handle-1', false, _, _) >> null and: 'module operations returns module and root nodes references' mockCpsModuleService.getRootNodeReferences(_,'cm-handle-1') >> ['some-module-1:some-root-node', 'some-module-2:some-root-node'] when: 'data sync poll is executed' - objectUnderTest.executeUnsynchronizedReadyCmHandleForInitialDataSync() + objectUnderTest.scheduledUnsynchronizedReadyCmHandleForInitialDataSync() then: 'cm handle state is retrieved' 1 * mockInventoryPersistence.getCmHandleState('cm-handle-1') >> compositeState and: 'first module sync succeeds' @@ -153,4 +195,8 @@ class DataSyncWatchdogSpec extends Specification { .build()).build()) return compositeState } + + def getLoggingEvent() { + return logAppender.list[0] + } } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdogSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdogSpec.groovy index 68aa6a1b6a..dd1c1fb571 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdogSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/impl/inventory/sync/ModuleSyncWatchdogSpec.groovy @@ -21,7 +21,14 @@ package org.onap.cps.ncmp.impl.inventory.sync +import ch.qos.logback.classic.Level +import ch.qos.logback.classic.Logger +import ch.qos.logback.classic.spi.ILoggingEvent +import ch.qos.logback.core.read.ListAppender import com.hazelcast.map.IMap +import org.onap.cps.init.actuator.ReadinessManager +import org.slf4j.LoggerFactory + import java.util.concurrent.ArrayBlockingQueue import org.onap.cps.ncmp.impl.inventory.models.YangModelCmHandle import spock.lang.Specification @@ -40,10 +47,39 @@ class ModuleSyncWatchdogSpec extends Specification { def mockCpsAndNcmpLock = Mock(IMap) - def objectUnderTest = new ModuleSyncWatchdog(mockModuleOperationsUtils, moduleSyncWorkQueue , mockModuleSyncStartedOnCmHandles, mockModuleSyncTasks, mockCpsAndNcmpLock) + def mockReadinessManager = Mock(ReadinessManager) + + def objectUnderTest = new ModuleSyncWatchdog(mockModuleOperationsUtils, moduleSyncWorkQueue , mockModuleSyncStartedOnCmHandles, mockModuleSyncTasks, mockCpsAndNcmpLock, mockReadinessManager) + + def logAppender = Spy(ListAppender) + + void setup() { + def logger = LoggerFactory.getLogger(ModuleSyncWatchdog) + logger.setLevel(Level.INFO) + logger.addAppender(logAppender) + logAppender.start() + } + + void cleanup() { + ((Logger) LoggerFactory.getLogger(ModuleSyncWatchdog.class)).detachAndStopAllAppenders() + } + + def 'Module sync watchdog is triggered'(){ + given: 'the system is not ready to accept traffic' + mockReadinessManager.isReady() >> false + when: 'module sync is started' + objectUnderTest.scheduledModuleSyncAdvisedCmHandles() + then: 'an event is logged with level INFO' + def loggingEvent = getLoggingEvent() + assert loggingEvent.level == Level.INFO + and: 'the log indicates that the system is not ready yet' + assert loggingEvent.formattedMessage == 'System is not ready yet' + } def 'Module sync advised cm handles with #scenario.'() { - given: 'module sync utilities returns #numberOfAdvisedCmHandles advised cm handles' + given: 'system is ready to accept traffic' + mockReadinessManager.isReady() >> true + and: 'module sync utilities returns #numberOfAdvisedCmHandles advised cm handles' mockModuleOperationsUtils.getAdvisedCmHandleIds() >> createCmHandleIds(numberOfAdvisedCmHandles) and: 'module sync utilities returns no failed (locked) cm handles' mockModuleOperationsUtils.getCmHandlesThatFailedModelSyncOrUpgrade() >> [] @@ -66,7 +102,9 @@ class ModuleSyncWatchdogSpec extends Specification { } def 'Module sync cm handles starts with no available threads.'() { - given: 'module sync utilities returns a advise cm handles' + given: 'system is ready to accept traffic' + mockReadinessManager.isReady() >> true + and: 'module sync utilities returns a advise cm handles' mockModuleOperationsUtils.getAdvisedCmHandleIds() >> createCmHandleIds(1) and: 'the work queue can be locked' mockCpsAndNcmpLock.tryLock('workQueueLock') >> true @@ -77,7 +115,9 @@ class ModuleSyncWatchdogSpec extends Specification { } def 'Module sync advised cm handle already handled by other thread.'() { - given: 'module sync utilities returns an advised cm handle' + given: 'system is ready to accept traffic' + mockReadinessManager.isReady() >> true + and: 'module sync utilities returns an advised cm handle' mockModuleOperationsUtils.getAdvisedCmHandleIds() >> createCmHandleIds(1) and: 'the work queue can be locked' mockCpsAndNcmpLock.tryLock('workQueueLock') >> true @@ -90,7 +130,9 @@ class ModuleSyncWatchdogSpec extends Specification { } def 'Module sync with previous cm handle(s) left in work queue.'() { - given: 'there is still a cm handle in the queue' + given: 'system is ready to accept traffic' + mockReadinessManager.isReady() >> true + and: 'there is still a cm handle in the queue' moduleSyncWorkQueue.offer('ch-1') when: 'module sync is started' objectUnderTest.moduleSyncAdvisedCmHandles() @@ -99,7 +141,9 @@ class ModuleSyncWatchdogSpec extends Specification { } def 'Reset failed cm handles.'() { - given: 'module sync utilities returns failed cm handles' + given: 'system is ready to accept traffic' + mockReadinessManager.isReady() >> true + and: 'module sync utilities returns failed cm handles' def failedCmHandles = [new YangModelCmHandle()] mockModuleOperationsUtils.getCmHandlesThatFailedModelSyncOrUpgrade() >> failedCmHandles when: 'reset failed cm handles is started' @@ -109,7 +153,9 @@ class ModuleSyncWatchdogSpec extends Specification { } def 'Module Sync Locking.'() { - given: 'module sync utilities returns an advised cm handle' + given: 'system is ready to accept traffic' + mockReadinessManager.isReady() >> true + and: 'module sync utilities returns an advised cm handle' mockModuleOperationsUtils.getAdvisedCmHandleIds() >> createCmHandleIds(1) and: 'can be locked is : #canLock' mockCpsAndNcmpLock.tryLock('workQueueLock') >> canLock @@ -128,4 +174,8 @@ class ModuleSyncWatchdogSpec extends Specification { def createCmHandleIds(numberOfCmHandles) { return (numberOfCmHandles > 0) ? (1..numberOfCmHandles).collect { 'ch-'+it } : [] } + + def getLoggingEvent() { + return logAppender.list[0] + } } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/CmDataSubscriptionModelLoaderSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/CmDataSubscriptionModelLoaderSpec.groovy index f1446adead..4d005bec7e 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/CmDataSubscriptionModelLoaderSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/CmDataSubscriptionModelLoaderSpec.groovy @@ -30,6 +30,7 @@ import org.onap.cps.api.CpsModuleService import org.onap.cps.api.model.Dataspace import org.onap.cps.init.actuator.ReadinessManager import org.slf4j.LoggerFactory +import org.springframework.boot.context.event.ApplicationReadyEvent import org.springframework.boot.context.event.ApplicationStartedEvent import org.springframework.context.annotation.AnnotationConfigApplicationContext import spock.lang.Specification @@ -65,11 +66,11 @@ class CmDataSubscriptionModelLoaderSpec extends Specification { applicationContext.close() } - def 'Onboard subscription model via application started event.'() { + def 'Onboard subscription model via application ready event.'() { given: 'dataspace is ready for use' mockCpsDataspaceService.getDataspace(NCMP_DATASPACE_NAME) >> new Dataspace('') - when: 'the application is started' - objectUnderTest.onApplicationEvent(Mock(ApplicationStartedEvent)) + when: 'the application is ready' + objectUnderTest.onApplicationEvent(Mock(ApplicationReadyEvent)) then: 'the module service to create schema set is called once' 1 * mockCpsModuleService.createSchemaSet(NCMP_DATASPACE_NAME, 'cm-data-job-subscriptions', expectedYangResourcesToContentMap) and: 'the admin service to create an anchor set is called once' diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/InventoryModelLoaderSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/InventoryModelLoaderSpec.groovy index 0a3894928e..5f42202cdf 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/InventoryModelLoaderSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/InventoryModelLoaderSpec.groovy @@ -32,6 +32,7 @@ import org.onap.cps.api.model.Dataspace import org.onap.cps.api.model.ModuleDefinition import org.onap.cps.init.actuator.ReadinessManager import org.slf4j.LoggerFactory +import org.springframework.boot.context.event.ApplicationReadyEvent import org.springframework.boot.context.event.ApplicationStartedEvent import org.springframework.context.ApplicationEventPublisher import org.springframework.context.annotation.AnnotationConfigApplicationContext @@ -79,8 +80,8 @@ class InventoryModelLoaderSpec extends Specification { mockCpsAdminService.getDataspace(NCMP_DATASPACE_NAME) >> new Dataspace('') and: 'module revision does not exist' mockCpsModuleService.getModuleDefinitionsByAnchorAndModule(_, _, _, _) >> Collections.emptyList() - when: 'the application is started' - objectUnderTest.onApplicationEvent(Mock(ApplicationStartedEvent)) + when: 'the application is ready' + objectUnderTest.onApplicationEvent(Mock(ApplicationReadyEvent)) then: 'the module service is used to create the new schema set from the correct resource' 1 * mockCpsModuleService.createSchemaSet(NCMP_DATASPACE_NAME, 'dmi-registry-2024-02-23', expectedPreviousYangResourceToContentMap) and: 'No schema sets are being removed by the module service (yet)' diff --git a/cps-service/src/main/java/org/onap/cps/init/AbstractModelLoader.java b/cps-service/src/main/java/org/onap/cps/init/AbstractModelLoader.java index 7bd3aaf1a6..563c2661c0 100644 --- a/cps-service/src/main/java/org/onap/cps/init/AbstractModelLoader.java +++ b/cps-service/src/main/java/org/onap/cps/init/AbstractModelLoader.java @@ -43,7 +43,7 @@ import org.onap.cps.api.parameters.CascadeDeleteAllowed; import org.onap.cps.init.actuator.ReadinessManager; import org.onap.cps.utils.JsonObjectMapper; import org.springframework.boot.SpringApplication; -import org.springframework.boot.context.event.ApplicationStartedEvent; +import org.springframework.boot.context.event.ApplicationReadyEvent; @Slf4j @RequiredArgsConstructor @@ -60,17 +60,15 @@ public abstract class AbstractModelLoader implements ModelLoader { private static final int EXIT_CODE_ON_ERROR = 1; @Override - public void onApplicationEvent(final ApplicationStartedEvent applicationStartedEvent) { - final String modelLoaderName = this.getClass().getSimpleName(); - readinessManager.registerStartupProcess(modelLoaderName); + public void onApplicationEvent(final ApplicationReadyEvent applicationReadyEvent) { try { onboardOrUpgradeModel(); } catch (final Exception exception) { log.error("Exiting application due to failure in onboarding model: {} ", exception.getMessage()); - exitApplication(applicationStartedEvent); + exitApplication(applicationReadyEvent); } finally { - readinessManager.markStartupProcessComplete(modelLoaderName); + readinessManager.markStartupProcessComplete(getName()); } } @@ -192,6 +190,11 @@ public abstract class AbstractModelLoader implements ModelLoader { } } + @Override + public String getName() { + return this.getClass().getSimpleName(); + } + /** * Checks if the specified revision of a module is installed. */ @@ -203,6 +206,7 @@ public abstract class AbstractModelLoader implements ModelLoader { return !moduleDefinitions.isEmpty(); } + Map mapYangResourcesToContent(final String... resourceNames) { final Map yangResourceContentByName = new HashMap<>(); for (final String resourceName: resourceNames) { @@ -221,7 +225,7 @@ public abstract class AbstractModelLoader implements ModelLoader { } } - private void exitApplication(final ApplicationStartedEvent applicationStartedEvent) { - SpringApplication.exit(applicationStartedEvent.getApplicationContext(), () -> EXIT_CODE_ON_ERROR); + private void exitApplication(final ApplicationReadyEvent applicationReadyEvent) { + SpringApplication.exit(applicationReadyEvent.getApplicationContext(), () -> EXIT_CODE_ON_ERROR); } } diff --git a/cps-service/src/main/java/org/onap/cps/init/ModelLoader.java b/cps-service/src/main/java/org/onap/cps/init/ModelLoader.java index 89c18b3159..c855787828 100644 --- a/cps-service/src/main/java/org/onap/cps/init/ModelLoader.java +++ b/cps-service/src/main/java/org/onap/cps/init/ModelLoader.java @@ -20,13 +20,15 @@ package org.onap.cps.init; -import org.springframework.boot.context.event.ApplicationStartedEvent; +import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.ApplicationListener; -public interface ModelLoader extends ApplicationListener { +public interface ModelLoader extends ApplicationListener { @Override - void onApplicationEvent(ApplicationStartedEvent applicationStartedEvent); + void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent); void onboardOrUpgradeModel(); + + String getName(); } diff --git a/cps-service/src/main/java/org/onap/cps/init/actuator/ModelLoaderRegistrationOnStartup.java b/cps-service/src/main/java/org/onap/cps/init/actuator/ModelLoaderRegistrationOnStartup.java new file mode 100644 index 0000000000..ae1b4f1fbe --- /dev/null +++ b/cps-service/src/main/java/org/onap/cps/init/actuator/ModelLoaderRegistrationOnStartup.java @@ -0,0 +1,54 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2025 OpenInfra Foundation Europe. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.init.actuator; + +import java.util.List; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.onap.cps.init.ModelLoader; +import org.springframework.boot.context.event.ApplicationStartedEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +@RequiredArgsConstructor +public class ModelLoaderRegistrationOnStartup implements ApplicationListener { + + private final ReadinessManager readinessManager; + // Spring will insert all concrete model loader classes here. + private final List modelLoaders; + + + /** + * Register the model loaders as part of the Application Started Phase. + * + * @param applicationStartedEvent Application Started Event + */ + @Override + public void onApplicationEvent(final ApplicationStartedEvent applicationStartedEvent) { + + modelLoaders.forEach(modelLoader -> { + log.info("Registering ModelLoader {}", modelLoader.getName()); + readinessManager.registerStartupProcess(modelLoader.getName()); + }); + } +} diff --git a/cps-service/src/main/java/org/onap/cps/init/actuator/ReadinessHealthIndicator.java b/cps-service/src/main/java/org/onap/cps/init/actuator/ReadinessStateHealthIndicatorConfig.java similarity index 60% rename from cps-service/src/main/java/org/onap/cps/init/actuator/ReadinessHealthIndicator.java rename to cps-service/src/main/java/org/onap/cps/init/actuator/ReadinessStateHealthIndicatorConfig.java index e40769aa07..288ca3e762 100644 --- a/cps-service/src/main/java/org/onap/cps/init/actuator/ReadinessHealthIndicator.java +++ b/cps-service/src/main/java/org/onap/cps/init/actuator/ReadinessStateHealthIndicatorConfig.java @@ -23,23 +23,29 @@ package org.onap.cps.init.actuator; import lombok.RequiredArgsConstructor; import org.springframework.boot.actuate.health.Health; import org.springframework.boot.actuate.health.HealthIndicator; -import org.springframework.stereotype.Component; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; -@Component +@Configuration @RequiredArgsConstructor -public class ReadinessHealthIndicator implements HealthIndicator { +public class ReadinessStateHealthIndicatorConfig { private final ReadinessManager readinessManager; - @Override - public Health health() { - if (readinessManager.isReady()) { - return Health.up() - .withDetail("Startup Processes", "All startup processes completed") - .build(); - } - return Health.down() - .withDetail("Startup Processes active", readinessManager.getStartupProcessesAsString()) - .build(); + /** + * Overriding the default Readiness State Health Indicator. + * + * @return Health Status ( UP or DOWN ) + */ + @Bean("readinessStateHealthIndicator") + public HealthIndicator readinessStateHealthIndicator() { + + return () -> { + if (readinessManager.isReady()) { + return Health.up().withDetail("Startup Processes", "All startup processes completed").build(); + } + return Health.down().withDetail("Startup Processes active", readinessManager.getStartupProcessesAsString()) + .build(); + }; } } diff --git a/cps-service/src/test/groovy/org/onap/cps/init/AbstractModelLoaderSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/init/AbstractModelLoaderSpec.groovy index ada76d58eb..097d79f6be 100644 --- a/cps-service/src/test/groovy/org/onap/cps/init/AbstractModelLoaderSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/init/AbstractModelLoaderSpec.groovy @@ -37,6 +37,7 @@ import org.onap.cps.api.exceptions.AlreadyDefinedException import org.onap.cps.init.actuator.ReadinessManager import org.slf4j.LoggerFactory import org.springframework.boot.SpringApplication +import org.springframework.boot.context.event.ApplicationReadyEvent import org.springframework.boot.context.event.ApplicationStartedEvent import org.springframework.context.annotation.AnnotationConfigApplicationContext import spock.lang.Specification @@ -69,18 +70,18 @@ class AbstractModelLoaderSpec extends Specification { loggingListAppender.stop() } - def 'Application started event triggers onboarding/upgrade'() { - when: 'Application (started) event is triggered' - objectUnderTest.onApplicationEvent(Mock(ApplicationStartedEvent)) + def 'Application ready event triggers onboarding/upgrade'() { + when: 'Application (ready) event is triggered' + objectUnderTest.onApplicationEvent(Mock(ApplicationReadyEvent)) then: 'the onboard/upgrade method is executed' 1 * objectUnderTest.onboardOrUpgradeModel() } - def 'Application started event handles startup exception'() { + def 'Application ready event handles startup exception'() { given: 'a startup exception is thrown during model onboarding' objectUnderTest.onboardOrUpgradeModel() >> { throw new ModelOnboardingException('test message','details are not logged') } - when: 'Application (started) event is triggered' - objectUnderTest.onApplicationEvent(new ApplicationStartedEvent(new SpringApplication(), null, applicationContext, null)) + when: 'Application (ready) event is triggered' + objectUnderTest.onApplicationEvent(new ApplicationReadyEvent(new SpringApplication(), null, applicationContext, null)) then: 'the exception message is logged' def logs = loggingListAppender.list.toString() assert logs.contains('test message') @@ -295,7 +296,12 @@ class AbstractModelLoaderSpec extends Specification { @Override void onboardOrUpgradeModel() { - // No operation needed for testing + // Not needed for testing + } + + @Override + String getName() { + // Not needed for testing } } } diff --git a/cps-service/src/test/groovy/org/onap/cps/init/CpsNotificationSubscriptionModelLoaderSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/init/CpsNotificationSubscriptionModelLoaderSpec.groovy index 9b399cc322..58bfd5e4e2 100644 --- a/cps-service/src/test/groovy/org/onap/cps/init/CpsNotificationSubscriptionModelLoaderSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/init/CpsNotificationSubscriptionModelLoaderSpec.groovy @@ -30,6 +30,7 @@ import org.onap.cps.api.CpsModuleService import org.onap.cps.api.model.Dataspace import org.onap.cps.init.actuator.ReadinessManager import org.slf4j.LoggerFactory +import org.springframework.boot.context.event.ApplicationReadyEvent import org.springframework.boot.context.event.ApplicationStartedEvent import org.springframework.context.annotation.AnnotationConfigApplicationContext import spock.lang.Specification @@ -68,11 +69,11 @@ class CpsNotificationSubscriptionModelLoaderSpec extends Specification { loggingListAppender.stop() } - def 'Onboard subscription model via application started event.'() { + def 'Onboard subscription model via application ready event.'() { given: 'dataspace is already present' mockCpsDataspaceService.getAllDataspaces() >> [new Dataspace('test')] when: 'the application is ready' - objectUnderTest.onApplicationEvent(Mock(ApplicationStartedEvent)) + objectUnderTest.onApplicationEvent(Mock(ApplicationReadyEvent)) then: 'the module service to create schema set is called once' 1 * mockCpsModuleService.createSchemaSet(CPS_DATASPACE_NAME, SCHEMASET_NAME, expectedYangResourcesToContents) and: 'the anchor service to create an anchor set is called once' diff --git a/cps-service/src/test/groovy/org/onap/cps/init/actuator/ModelLoaderRegistrationOnStartupSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/init/actuator/ModelLoaderRegistrationOnStartupSpec.groovy new file mode 100644 index 0000000000..b3cbf04912 --- /dev/null +++ b/cps-service/src/test/groovy/org/onap/cps/init/actuator/ModelLoaderRegistrationOnStartupSpec.groovy @@ -0,0 +1,48 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2025 OpenInfra Foundation Europe. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.init.actuator + +import org.onap.cps.init.ModelLoader +import org.springframework.boot.context.event.ApplicationStartedEvent +import spock.lang.Specification + +class ModelLoaderRegistrationOnStartupSpec extends Specification { + + def mockReadinessManager = Mock(ReadinessManager) + def mockModelLoader1 = Mock(ModelLoader) + def mockModelLoader2 = Mock(ModelLoader) + def mockApplicationStartedEvent = Mock(ApplicationStartedEvent) + + def objectUnderTest = new ModelLoaderRegistrationOnStartup(mockReadinessManager, [mockModelLoader1, mockModelLoader2]) + + def 'Register the model loaders via application started event'() { + given: 'model loaders with specific name' + mockModelLoader1.name >> 'my-loader-1' + mockModelLoader2.name >> 'my-loader-2' + when: ' application started event is fired' + objectUnderTest.onApplicationEvent(mockApplicationStartedEvent) + then: 'loaders are registered with the readiness managers' + 1 * mockReadinessManager.registerStartupProcess('my-loader-1') + 1 * mockReadinessManager.registerStartupProcess('my-loader-2') + } + + +} diff --git a/cps-service/src/test/groovy/org/onap/cps/init/actuator/ReadinessHealthIndicatorSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/init/actuator/ReadinessStateHealthIndicatorConfigSpec.groovy similarity index 84% rename from cps-service/src/test/groovy/org/onap/cps/init/actuator/ReadinessHealthIndicatorSpec.groovy rename to cps-service/src/test/groovy/org/onap/cps/init/actuator/ReadinessStateHealthIndicatorConfigSpec.groovy index 3091578905..aed93791a7 100644 --- a/cps-service/src/test/groovy/org/onap/cps/init/actuator/ReadinessHealthIndicatorSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/init/actuator/ReadinessStateHealthIndicatorConfigSpec.groovy @@ -22,15 +22,15 @@ package org.onap.cps.init.actuator import spock.lang.Specification; -class ReadinessHealthIndicatorSpec extends Specification { +class ReadinessStateHealthIndicatorConfigSpec extends Specification { def readinessManager = new ReadinessManager() - def objectUnderTest = new ReadinessHealthIndicator(readinessManager) + def objectUnderTest = new ReadinessStateHealthIndicatorConfig(readinessManager) def 'CPS service UP when all loaders are completed'() { given: 'no loaders are in progress' when: 'cps health check is invoked' - def cpsHealth = objectUnderTest.health() + def cpsHealth = objectUnderTest.readinessStateHealthIndicator().getHealth(true) then: 'health status is UP with following message' assert cpsHealth.status.code == 'UP' assert cpsHealth.details['Startup Processes'] == 'All startup processes completed' @@ -40,7 +40,7 @@ class ReadinessHealthIndicatorSpec extends Specification { given: 'any module loader is still running' readinessManager.registerStartupProcess('someLoader') when: 'cps health check is invoked' - def cpsHealth = objectUnderTest.health() + def cpsHealth = objectUnderTest.readinessStateHealthIndicator().getHealth(true) then: 'cps service is DOWN with loaders listed' assert cpsHealth.status.code == 'DOWN' def busyLoaders = cpsHealth.details['Startup Processes active'] @@ -52,7 +52,7 @@ class ReadinessHealthIndicatorSpec extends Specification { readinessManager.registerStartupProcess('someLoader') when: 'module loader completes' readinessManager.markStartupProcessComplete('someLoader') - def health = objectUnderTest.health() + def health = objectUnderTest.readinessStateHealthIndicator().getHealth(true) then: 'cps health status flips to UP' assert health.status.code == 'UP' assert health.details['Startup Processes'] == 'All startup processes completed' diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy index d8c1e16848..e7312f8990 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy @@ -33,6 +33,8 @@ import org.onap.cps.api.CpsModuleService import org.onap.cps.api.CpsQueryService import org.onap.cps.api.exceptions.DataspaceNotFoundException import org.onap.cps.api.model.DataNode +import org.onap.cps.init.actuator.ModelLoaderRegistrationOnStartup +import org.onap.cps.init.actuator.ReadinessManager import org.onap.cps.integration.DatabaseTestContainer import org.onap.cps.integration.KafkaTestContainer import org.onap.cps.ncmp.api.inventory.models.CmHandleState @@ -60,7 +62,6 @@ import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMock import org.springframework.boot.test.context.SpringBootTest import org.springframework.context.annotation.ComponentScan import org.springframework.data.jpa.repository.config.EnableJpaRepositories -import org.springframework.test.context.ActiveProfiles import org.springframework.test.web.servlet.MockMvc import org.testcontainers.spock.Testcontainers import spock.lang.Shared @@ -77,7 +78,6 @@ import java.util.concurrent.BlockingQueue @EnableJpaRepositories(basePackageClasses = [DataspaceRepository]) @ComponentScan(basePackages = ['org.onap.cps']) @EntityScan('org.onap.cps.ri.models') -@ActiveProfiles('module-sync-delayed') abstract class CpsIntegrationSpecBase extends Specification { static KafkaConsumer kafkaConsumer @@ -157,6 +157,12 @@ abstract class CpsIntegrationSpecBase extends Specification { @Autowired AlternateIdMatcher alternateIdMatcher + @Autowired + ModelLoaderRegistrationOnStartup modelLoaderRegistrationOnStartup + + @Autowired + ReadinessManager readinessManager + @Value('${ncmp.policy-executor.server.port:8080}') private String policyServerPort; @@ -197,8 +203,10 @@ abstract class CpsIntegrationSpecBase extends Specification { mockPolicyServer.setDispatcher(policyDispatcher) mockPolicyServer.start(Integer.valueOf(policyServerPort)) - DMI1_URL = String.format("http://%s:%s", mockDmiServer1.getHostName(), mockDmiServer1.getPort()) - DMI2_URL = String.format("http://%s:%s", mockDmiServer2.getHostName(), mockDmiServer2.getPort()) + DMI1_URL = String.format('http://%s:%s', mockDmiServer1.getHostName(), mockDmiServer1.getPort()) + DMI2_URL = String.format('http://%s:%s', mockDmiServer2.getHostName(), mockDmiServer2.getPort()) + + readinessManager.registerStartupProcess('Dummy process to prevent watchdogs processes starting during integration tests') } def cleanup() { diff --git a/integration-test/src/test/resources/application-module-sync-delayed.yml b/integration-test/src/test/resources/application-module-sync-delayed.yml deleted file mode 100644 index 27c99e93b2..0000000000 --- a/integration-test/src/test/resources/application-module-sync-delayed.yml +++ /dev/null @@ -1,23 +0,0 @@ - -# ============LICENSE_START======================================================= -# Copyright (C) 2024 Nordix Foundation. -# ================================================================================ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# ============LICENSE_END========================================================= -test: - ncmp: - timers: - advised-modules-sync: - initial-delay-ms: 600000 - diff --git a/integration-test/src/test/resources/application.yml b/integration-test/src/test/resources/application.yml index cb60de84b2..efe595ed7f 100644 --- a/integration-test/src/test/resources/application.yml +++ b/integration-test/src/test/resources/application.yml @@ -173,7 +173,6 @@ ncmp: timers: advised-modules-sync: - initial-delay-ms: 0 sleep-time-ms: 1000000 cm-handle-data-sync: sleep-time-ms: 30000