Add ReadinessHealthIndicator to expose migration readiness status 69/141969/6
authorsourabh_sourabh <sourabh.sourabh@est.tech>
Tue, 2 Sep 2025 16:22:06 +0000 (17:22 +0100)
committersourabh_sourabh <sourabh.sourabh@est.tech>
Thu, 4 Sep 2025 10:47:26 +0000 (11:47 +0100)
- Reports READY when migrations are complete.
- Reports NOT READY when a migration or rollback is in progress.
- This will allow Helm (Kubernetes) and other monitoring tools to detect application readiness during migrations and act accordingly.

Issue-ID: CPS-2974
Change-Id: I72441e186178d4cbd0cd98e754ca6059b4ff0bb6
Signed-off-by: sourabh_sourabh <sourabh.sourabh@est.tech>
14 files changed:
cps-charts/values.yaml
cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/CmDataSubscriptionModelLoader.java
cps-ncmp-service/src/main/java/org/onap/cps/ncmp/init/InventoryModelLoader.java
cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/CmDataSubscriptionModelLoaderSpec.groovy
cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/init/InventoryModelLoaderSpec.groovy
cps-service/pom.xml
cps-service/src/main/java/org/onap/cps/init/AbstractModelLoader.java
cps-service/src/main/java/org/onap/cps/init/CpsNotificationSubscriptionModelLoader.java
cps-service/src/main/java/org/onap/cps/init/actuator/ReadinessHealthIndicator.java [new file with mode: 0644]
cps-service/src/main/java/org/onap/cps/init/actuator/ReadinessManager.java [new file with mode: 0644]
cps-service/src/test/groovy/org/onap/cps/init/AbstractModelLoaderSpec.groovy
cps-service/src/test/groovy/org/onap/cps/init/CpsNotificationSubscriptionModelLoaderSpec.groovy
cps-service/src/test/groovy/org/onap/cps/init/actuator/ReadinessHealthIndicatorSpec.groovy [new file with mode: 0644]
cps-service/src/test/groovy/org/onap/cps/init/actuator/ReadinessManagerSpec.groovy [new file with mode: 0644]

index bbe8ee3..e1773d2 100644 (file)
@@ -47,6 +47,7 @@ cps:
     CPS_MONITORING_MICROMETER_JVM_EXTRAS: "true"
     JAVA_TOOL_OPTIONS: "-XX:InitialRAMPercentage=70.0 -XX:MaxRAMPercentage=70.0"
     HAZELCAST_MODE_KUBERNETES_ENABLED: "true"
+    NCMP_INVENTORY_MODEL_UPGRADE_R20250722_ENABLED: 'false'
 
 kafka:
   enabled: true
index 4b8a95c..278332d 100644 (file)
@@ -28,6 +28,7 @@ import org.onap.cps.api.CpsDataService;
 import org.onap.cps.api.CpsDataspaceService;
 import org.onap.cps.api.CpsModuleService;
 import org.onap.cps.init.AbstractModelLoader;
+import org.onap.cps.init.actuator.ReadinessManager;
 import org.springframework.stereotype.Service;
 
 @Slf4j
@@ -40,9 +41,11 @@ public class CmDataSubscriptionModelLoader extends AbstractModelLoader {
     private static final String REGISTRY_DATA_NODE_NAME = "dataJob";
 
     public CmDataSubscriptionModelLoader(final CpsDataspaceService cpsDataspaceService,
-            final CpsModuleService cpsModuleService, final CpsAnchorService cpsAnchorService,
-            final CpsDataService cpsDataService) {
-        super(cpsDataspaceService, cpsModuleService, cpsAnchorService, cpsDataService);
+                                         final CpsModuleService cpsModuleService,
+                                         final CpsAnchorService cpsAnchorService,
+                                         final CpsDataService cpsDataService,
+                                         final ReadinessManager readinessManager) {
+        super(cpsDataspaceService, cpsModuleService, cpsAnchorService, cpsDataService, readinessManager);
     }
 
     @Override
index 51ba3ec..7ca52f1 100644 (file)
@@ -30,6 +30,7 @@ import org.onap.cps.api.CpsDataService;
 import org.onap.cps.api.CpsDataspaceService;
 import org.onap.cps.api.CpsModuleService;
 import org.onap.cps.init.AbstractModelLoader;
+import org.onap.cps.init.actuator.ReadinessManager;
 import org.onap.cps.ncmp.utils.events.NcmpInventoryModelOnboardingFinishedEvent;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.ApplicationEventPublisher;
@@ -40,6 +41,7 @@ import org.springframework.stereotype.Service;
 public class InventoryModelLoader extends AbstractModelLoader {
 
     private final ApplicationEventPublisher applicationEventPublisher;
+
     private static final String PREVIOUS_SCHEMA_SET_NAME = "dmi-registry-2024-02-23";
     private static final String NEW_INVENTORY_SCHEMA_SET_NAME = "dmi-registry-2025-07-22";
     private static final String INVENTORY_YANG_MODULE_NAME = "dmi-registry";
@@ -47,12 +49,17 @@ public class InventoryModelLoader extends AbstractModelLoader {
     @Value("${ncmp.inventory.model.upgrade.r20250722.enabled:false}")
     private boolean newRevisionEnabled;
 
+    /**
+     * Creates a new {@code InventoryModelLoader} instance responsible for onboarding or upgrading
+     * the NCMP inventory model schema sets and managing readiness state during migration.
+     */
     public InventoryModelLoader(final CpsDataspaceService cpsDataspaceService,
                                 final CpsModuleService cpsModuleService,
                                 final CpsAnchorService cpsAnchorService,
                                 final CpsDataService cpsDataService,
-                                final ApplicationEventPublisher applicationEventPublisher) {
-        super(cpsDataspaceService, cpsModuleService, cpsAnchorService, cpsDataService);
+                                final ApplicationEventPublisher applicationEventPublisher,
+                                final ReadinessManager readinessManager) {
+        super(cpsDataspaceService, cpsModuleService, cpsAnchorService, cpsDataService, readinessManager);
         this.applicationEventPublisher = applicationEventPublisher;
     }
 
@@ -107,6 +114,24 @@ public class InventoryModelLoader extends AbstractModelLoader {
         // TODO further implementation is pending
         //1. Load all the cm handles (in batch)
         //2. Copy the state and known properties
+        log.info("Starting inventory module data migration...");
+
+        // Simulate a 4-minute migration (240 seconds total)
+        final int totalSeconds = 240;
+        final int stepSeconds = 30; // log progress every 30 seconds
+        final int steps = totalSeconds / stepSeconds;
+
+        for (int i = 1; i <= steps; i++) {
+            try {
+                Thread.sleep(stepSeconds * 1000L);
+            } catch (final InterruptedException e) {
+                Thread.currentThread().interrupt();
+                log.warn("Migration interrupted!", e);
+                return;
+            }
+            final int progress = (i * 100) / steps;
+            log.info("Migration progress: {}%", progress);
+        }
         log.info("Inventory module data migration is completed successfully.");
     }
 
index d26afb4..0a84f51 100644 (file)
@@ -28,6 +28,7 @@ import org.onap.cps.api.CpsDataService
 import org.onap.cps.api.CpsDataspaceService
 import org.onap.cps.api.CpsModuleService
 import org.onap.cps.api.model.Dataspace
+import org.onap.cps.init.actuator.ReadinessManager
 import org.slf4j.LoggerFactory
 import org.springframework.boot.context.event.ApplicationStartedEvent
 import org.springframework.context.annotation.AnnotationConfigApplicationContext
@@ -41,7 +42,8 @@ class CmDataSubscriptionModelLoaderSpec extends Specification {
     def mockCpsModuleService = Mock(CpsModuleService)
     def mockCpsDataService = Mock(CpsDataService)
     def mockCpsAnchorService = Mock(CpsAnchorService)
-    def objectUnderTest = new CmDataSubscriptionModelLoader(mockCpsDataspaceService, mockCpsModuleService, mockCpsAnchorService, mockCpsDataService)
+    def mockReadinessManager = Mock(ReadinessManager)
+    def objectUnderTest = new CmDataSubscriptionModelLoader(mockCpsDataspaceService, mockCpsModuleService, mockCpsAnchorService, mockCpsDataService, mockReadinessManager)
 
     def applicationContext = new AnnotationConfigApplicationContext()
 
index dfc7c10..2ae66c9 100644 (file)
@@ -30,6 +30,7 @@ import org.onap.cps.api.CpsModuleService
 import org.onap.cps.api.exceptions.AnchorNotFoundException
 import org.onap.cps.api.model.Dataspace
 import org.onap.cps.api.model.ModuleDefinition
+import org.onap.cps.init.actuator.ReadinessManager
 import org.slf4j.LoggerFactory
 import org.springframework.boot.context.event.ApplicationStartedEvent
 import org.springframework.context.ApplicationEventPublisher
@@ -46,7 +47,8 @@ class InventoryModelLoaderSpec extends Specification {
     def mockCpsDataService = Mock(CpsDataService)
     def mockCpsAnchorService = Mock(CpsAnchorService)
     def mockApplicationEventPublisher = Mock(ApplicationEventPublisher)
-    def objectUnderTest = new InventoryModelLoader(mockCpsAdminService, mockCpsModuleService, mockCpsAnchorService, mockCpsDataService, mockApplicationEventPublisher)
+    def mockReadinessManager = Mock(ReadinessManager)
+    def objectUnderTest = new InventoryModelLoader(mockCpsAdminService, mockCpsModuleService, mockCpsAnchorService, mockCpsDataService, mockApplicationEventPublisher, mockReadinessManager)
 
     def applicationContext = new AnnotationConfigApplicationContext()
 
index de0d558..8b7de2c 100644 (file)
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
   ============LICENSE_START=======================================================
-  Copyright (C) 2021-2024 Nordix Foundation
+  Copyright (C) 2021-2025 OpenInfra Foundation Europe. All rights reserved.
   Modifications Copyright (C) 2021 Bell Canada.
   Modifications Copyright (C) 2021 Pantheon.tech
   Modifications Copyright (C) 2022 Deutsche Telekom AG
       <groupId>org.springframework.boot</groupId>
       <artifactId>spring-boot-starter-aop</artifactId>
     </dependency>
+      <dependency>
+          <groupId>org.springframework.boot</groupId>
+          <artifactId>spring-boot-starter-actuator</artifactId>
+      </dependency>
     <dependency>
       <groupId>org.springframework.boot</groupId>
       <artifactId>spring-boot-starter-validation</artifactId>
index a87c36e..f1bf78a 100644 (file)
@@ -40,6 +40,7 @@ import org.onap.cps.api.exceptions.DuplicatedYangResourceException;
 import org.onap.cps.api.exceptions.ModelOnboardingException;
 import org.onap.cps.api.model.ModuleDefinition;
 import org.onap.cps.api.parameters.CascadeDeleteAllowed;
+import org.onap.cps.init.actuator.ReadinessManager;
 import org.onap.cps.utils.JsonObjectMapper;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.context.event.ApplicationStartedEvent;
@@ -52,6 +53,7 @@ public abstract class AbstractModelLoader implements ModelLoader {
     private final CpsModuleService cpsModuleService;
     protected final CpsAnchorService cpsAnchorService;
     protected final CpsDataService cpsDataService;
+    protected final ReadinessManager readinessManager;
 
     private final JsonObjectMapper jsonObjectMapper = new JsonObjectMapper(new ObjectMapper());
 
@@ -59,12 +61,16 @@ public abstract class AbstractModelLoader implements ModelLoader {
 
     @Override
     public void onApplicationEvent(final ApplicationStartedEvent applicationStartedEvent) {
+        final String modelLoaderName = this.getClass().getSimpleName();
+        readinessManager.registerStartupProcess(modelLoaderName);
         try {
             onboardOrUpgradeModel();
         } catch (final Exception exception) {
             log.error("Exiting application due to failure in onboarding model: {} ",
-                exception.getMessage());
+                    exception.getMessage());
             exitApplication(applicationStartedEvent);
+        } finally {
+            readinessManager.markStartupProcessComplete(modelLoaderName);
         }
     }
 
index bf60f8d..ee26761 100644 (file)
@@ -25,6 +25,7 @@ import org.onap.cps.api.CpsAnchorService;
 import org.onap.cps.api.CpsDataService;
 import org.onap.cps.api.CpsDataspaceService;
 import org.onap.cps.api.CpsModuleService;
+import org.onap.cps.init.actuator.ReadinessManager;
 import org.springframework.stereotype.Service;
 
 @Slf4j
@@ -40,8 +41,9 @@ public class CpsNotificationSubscriptionModelLoader extends AbstractModelLoader
     public CpsNotificationSubscriptionModelLoader(final CpsDataspaceService cpsDataspaceService,
                                                   final CpsModuleService cpsModuleService,
                                                   final CpsAnchorService cpsAnchorService,
-                                                  final CpsDataService cpsDataService) {
-        super(cpsDataspaceService, cpsModuleService, cpsAnchorService, cpsDataService);
+                                                  final CpsDataService cpsDataService,
+                                                  final ReadinessManager readinessManager) {
+        super(cpsDataspaceService, cpsModuleService, cpsAnchorService, cpsDataService, readinessManager);
     }
 
     @Override
diff --git a/cps-service/src/main/java/org/onap/cps/init/actuator/ReadinessHealthIndicator.java b/cps-service/src/main/java/org/onap/cps/init/actuator/ReadinessHealthIndicator.java
new file mode 100644 (file)
index 0000000..caf430b
--- /dev/null
@@ -0,0 +1,46 @@
+/*
+ * ============LICENSE_START=======================================================
+ * Copyright (C) 2025 OpenInfra Foundation Europe. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.init.actuator;
+
+import lombok.RequiredArgsConstructor;
+import org.springframework.boot.actuate.health.Health;
+import org.springframework.boot.actuate.health.HealthIndicator;
+import org.springframework.stereotype.Component;
+
+@Component
+@RequiredArgsConstructor
+public class ReadinessHealthIndicator implements HealthIndicator {
+
+    private final ReadinessManager readinessManager;
+
+    @Override
+    public Health health() {
+        if (readinessManager.isReady()) {
+            return Health.up()
+                    .withDetail("Startup Processes", "All startup processes completed")
+                    .build();
+        } else {
+            return Health.down()
+                    .withDetail("Startup Processes active", readinessManager.getStartupProcessesAsString())
+                    .build();
+        }
+    }
+}
diff --git a/cps-service/src/main/java/org/onap/cps/init/actuator/ReadinessManager.java b/cps-service/src/main/java/org/onap/cps/init/actuator/ReadinessManager.java
new file mode 100644 (file)
index 0000000..ed70ddb
--- /dev/null
@@ -0,0 +1,47 @@
+/*
+ * ============LICENSE_START=======================================================
+ * Copyright (C) 2025 OpenInfra Foundation Europe. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.init.actuator;
+
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ReadinessManager {
+
+    private final Set<String> startupProcesses = ConcurrentHashMap.newKeySet();
+
+    public void registerStartupProcess(final String name) {
+        startupProcesses.add(name);
+    }
+
+    public void markStartupProcessComplete(final String name) {
+        startupProcesses.remove(name);
+    }
+
+    public String getStartupProcessesAsString() {
+        return String.join(", ", startupProcesses);
+    }
+
+    public boolean isReady() {
+        return startupProcesses.isEmpty();
+    }
+}
index 8692e60..ada76d5 100644 (file)
@@ -34,6 +34,7 @@ import org.onap.cps.api.exceptions.ModelOnboardingException
 import org.onap.cps.api.model.ModuleDefinition
 import org.onap.cps.api.parameters.CascadeDeleteAllowed
 import org.onap.cps.api.exceptions.AlreadyDefinedException
+import org.onap.cps.init.actuator.ReadinessManager
 import org.slf4j.LoggerFactory
 import org.springframework.boot.SpringApplication
 import org.springframework.boot.context.event.ApplicationStartedEvent
@@ -46,7 +47,8 @@ class AbstractModelLoaderSpec extends Specification {
     def mockCpsModuleService = Mock(CpsModuleService)
     def mockCpsDataService = Mock(CpsDataService)
     def mockCpsAnchorService = Mock(CpsAnchorService)
-    def objectUnderTest = Spy(new TestModelLoader(mockCpsDataspaceService, mockCpsModuleService, mockCpsAnchorService, mockCpsDataService))
+    def mockReadinessManager = Mock(ReadinessManager)
+    def objectUnderTest = Spy(new TestModelLoader(mockCpsDataspaceService, mockCpsModuleService, mockCpsAnchorService, mockCpsDataService, mockReadinessManager))
 
     def applicationContext = new AnnotationConfigApplicationContext()
 
@@ -286,8 +288,9 @@ class AbstractModelLoaderSpec extends Specification {
         TestModelLoader(final CpsDataspaceService cpsDataspaceService,
                         final CpsModuleService cpsModuleService,
                         final CpsAnchorService cpsAnchorService,
-                        final CpsDataService cpsDataService) {
-            super(cpsDataspaceService, cpsModuleService, cpsAnchorService, cpsDataService)
+                        final CpsDataService cpsDataService,
+                        final ReadinessManager readinessManager) {
+            super(cpsDataspaceService, cpsModuleService, cpsAnchorService, cpsDataService, readinessManager)
         }
 
         @Override
index 1e2dc54..9b399cc 100644 (file)
@@ -28,6 +28,7 @@ import org.onap.cps.api.CpsDataService
 import org.onap.cps.api.CpsDataspaceService
 import org.onap.cps.api.CpsModuleService
 import org.onap.cps.api.model.Dataspace
+import org.onap.cps.init.actuator.ReadinessManager
 import org.slf4j.LoggerFactory
 import org.springframework.boot.context.event.ApplicationStartedEvent
 import org.springframework.context.annotation.AnnotationConfigApplicationContext
@@ -38,7 +39,8 @@ class CpsNotificationSubscriptionModelLoaderSpec extends Specification {
     def mockCpsModuleService = Mock(CpsModuleService)
     def mockCpsDataService = Mock(CpsDataService)
     def mockCpsAnchorService = Mock(CpsAnchorService)
-    def objectUnderTest = new CpsNotificationSubscriptionModelLoader(mockCpsDataspaceService, mockCpsModuleService, mockCpsAnchorService, mockCpsDataService)
+    def mockReadinessManager = Mock(ReadinessManager)
+    def objectUnderTest = new CpsNotificationSubscriptionModelLoader(mockCpsDataspaceService, mockCpsModuleService, mockCpsAnchorService, mockCpsDataService, mockReadinessManager)
 
     def applicationContext = new AnnotationConfigApplicationContext()
 
diff --git a/cps-service/src/test/groovy/org/onap/cps/init/actuator/ReadinessHealthIndicatorSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/init/actuator/ReadinessHealthIndicatorSpec.groovy
new file mode 100644 (file)
index 0000000..3091578
--- /dev/null
@@ -0,0 +1,60 @@
+/*
+ * ============LICENSE_START=======================================================
+ * Copyright (C) 2025 OpenInfra Foundation Europe. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.init.actuator
+
+import spock.lang.Specification;
+
+class ReadinessHealthIndicatorSpec extends Specification {
+
+    def readinessManager = new ReadinessManager()
+    def objectUnderTest = new ReadinessHealthIndicator(readinessManager)
+
+    def 'CPS service UP when all loaders are completed'() {
+        given: 'no loaders are in progress'
+        when: 'cps health check is invoked'
+            def cpsHealth = objectUnderTest.health()
+        then: 'health status is UP with following message'
+            assert cpsHealth.status.code == 'UP'
+            assert cpsHealth.details['Startup Processes'] == 'All startup processes completed'
+    }
+
+    def 'CPS service is DOWN when any loader is in progress'() {
+        given: 'any module loader is still running'
+            readinessManager.registerStartupProcess('someLoader')
+        when: 'cps health check is invoked'
+            def cpsHealth = objectUnderTest.health()
+        then: 'cps service is DOWN with loaders listed'
+            assert cpsHealth.status.code == 'DOWN'
+            def busyLoaders = cpsHealth.details['Startup Processes active']
+            assert busyLoaders.contains('someLoader')
+    }
+
+    def 'CPS service is UP after loaders complete'() {
+        given: 'a loader in progress'
+            readinessManager.registerStartupProcess('someLoader')
+        when: 'module loader completes'
+            readinessManager.markStartupProcessComplete('someLoader')
+            def health = objectUnderTest.health()
+        then: 'cps health status flips to UP'
+            assert health.status.code == 'UP'
+            assert health.details['Startup Processes'] == 'All startup processes completed'
+    }
+}
diff --git a/cps-service/src/test/groovy/org/onap/cps/init/actuator/ReadinessManagerSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/init/actuator/ReadinessManagerSpec.groovy
new file mode 100644 (file)
index 0000000..3db1eb2
--- /dev/null
@@ -0,0 +1,57 @@
+/*
+ * ============LICENSE_START=======================================================
+ * Copyright (C) 2025 OpenInfra Foundation Europe. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.init.actuator
+
+import spock.lang.Specification;
+
+class ReadinessManagerSpec extends Specification {
+
+    def readinessManager = new ReadinessManager()
+
+    def 'Default readiness state'() {
+        expect: 'by default, cps service is Up and running'
+            assert readinessManager.isReady() == true
+    }
+
+    def 'Readiness state during model loading'() {
+        when: 'some loader process is registered'
+            readinessManager.registerStartupProcess('someLoader')
+        then: 'readiness manager report system is NOT ready'
+            assert readinessManager.isReady() == false
+            assert readinessManager.getStartupProcessesAsString() == 'someLoader'
+    }
+
+    def 'Readiness state transitions'() {
+        given: 'multiple loader processes are registered'
+            readinessManager.registerStartupProcess('someLoader-1')
+            readinessManager.registerStartupProcess('someLoader-2')
+        when: 'one process completes'
+            readinessManager.markStartupProcessComplete('someLoader-1')
+        then: 'still system is reposted as NOT READY with active loader name'
+            assert readinessManager.isReady() == false
+            assert readinessManager.getStartupProcessesAsString() == 'someLoader-2'
+        when: 'the last process completes'
+            readinessManager.markStartupProcessComplete('someLoader-2')
+        then: 'all processes completed, service is ready without any active loader'
+            assert readinessManager.isReady() == true
+            assert readinessManager.getStartupProcessesAsString() == ''
+    }
+}