From 722701b8893e0f3c27aa63ab506a32b891dd64d0 Mon Sep 17 00:00:00 2001 From: DylanB95EST Date: Mon, 20 Jun 2022 13:45:19 +0100 Subject: [PATCH] Retry Module-Sync based on from last failure Retry algorithm for module-sync based on last failure Poll Lock Reason, check if lock has surpassed minimum time based on last update time and lock reason Issue-ID: CPS-1076 Change-Id: Ifbbabd2b403f88f1bbe3fae3f125b1e9cb2091aa Signed-off-by: DylanB95EST --- .../ncmp/api/inventory/CompositeStateBuilder.java | 5 +++-- .../api/inventory/sync/ModuleSyncWatchdog.java | 15 +++++++------ .../cps/ncmp/api/inventory/sync/SyncUtils.java | 25 ++++++++++++++++++++++ .../ncmp/api/inventory/sync/ModuleSyncSpec.groovy | 17 +++++++++++---- .../ncmp/api/inventory/sync/SyncUtilsSpec.groovy | 22 +++++++++++++++++++ 5 files changed, 72 insertions(+), 12 deletions(-) diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/CompositeStateBuilder.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/CompositeStateBuilder.java index 012ba5ede..91e92ea6f 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/CompositeStateBuilder.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/CompositeStateBuilder.java @@ -111,8 +111,9 @@ public class CompositeStateBuilder { * @return CompositeState */ public CompositeStateBuilder fromDataNode(final DataNode dataNode) { - this.cmHandleState = CmHandleState.valueOf((String) dataNode.getLeaves() - .get("cm-handle-state")); + this.cmHandleState = CmHandleState.valueOf((String) dataNode.getLeaves() + .get("cm-handle-state")); + this.lastUpdatedTime = (String) dataNode.getLeaves().get("last-update-time"); for (final DataNode stateChildNode : dataNode.getChildDataNodes()) { if (stateChildNode.getXpath().endsWith("/lock-reason")) { this.lockReason = getLockReason(stateChildNode); diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncWatchdog.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncWatchdog.java index e590ca1cd..402f9f6b4 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncWatchdog.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncWatchdog.java @@ -74,13 +74,16 @@ public class ModuleSyncWatchdog { * Execute Cm Handle poll which changes the cm handle state from 'LOCKED' to 'ADVISED'. */ @Scheduled(fixedDelayString = "${timers.locked-modules-sync.sleep-time-ms:300000}") - public void executeLockedMisbehavingCmHandlePoll() { + public void executeLockedCmHandlePoll() { final List lockedMisbehavingCmHandles = syncUtils.getLockedMisbehavingYangModelCmHandles(); - for (final YangModelCmHandle lockedMisbehavingModelCmHandle : lockedMisbehavingCmHandles) { - final CompositeState compositeState = lockedMisbehavingModelCmHandle.getCompositeState(); - setCompositeStateToAdvisedAndRetainOldLockReasonDetails(compositeState); - log.debug("Locked misbehaving cm handle {} is being recycled", lockedMisbehavingModelCmHandle.getId()); - inventoryPersistence.saveCmHandleState(lockedMisbehavingModelCmHandle.getId(), compositeState); + for (final YangModelCmHandle moduleSyncFailedCmHandle : lockedMisbehavingCmHandles) { + final CompositeState compositeState = moduleSyncFailedCmHandle.getCompositeState(); + final boolean isReadyForRetry = syncUtils.isReadyForRetry(compositeState); + if (isReadyForRetry) { + setCompositeStateToAdvisedAndRetainOldLockReasonDetails(compositeState); + log.debug("Locked misbehaving cm handle {} is being recycled", moduleSyncFailedCmHandle.getId()); + inventoryPersistence.saveCmHandleState(moduleSyncFailedCmHandle.getId(), compositeState); + } } } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/sync/SyncUtils.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/sync/SyncUtils.java index 42edcb7ec..8b7dfe6b8 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/sync/SyncUtils.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/sync/SyncUtils.java @@ -24,6 +24,9 @@ package org.onap.cps.ncmp.api.inventory.sync; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import java.security.SecureRandom; +import java.time.Duration; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -140,6 +143,28 @@ public class SyncUtils { .lockReasonCategory(lockReasonCategory).build()); } + + /** + * Check if the retry mechanism should attempt to unlock the cm handle based on the last update time. + * + * @param compositeState the composite state currently in the locked state + * @return if the retry mechanism should be attempted + */ + public boolean isReadyForRetry(final CompositeState compositeState) { + int timeUntilNextAttempt = 1; + final OffsetDateTime time = + OffsetDateTime.parse(compositeState.getLastUpdateTime(), + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ")); + final Matcher matcher = retryAttemptPattern.matcher(compositeState.getLockReason().getDetails()); + if (matcher.find()) { + timeUntilNextAttempt = (int) Math.pow(2, Integer.parseInt(matcher.group(1))); + } else { + log.debug("First Attempt: no current attempts found."); + } + final int timeSinceLastAttempt = (int) Duration.between(time, OffsetDateTime.now()).toMinutes(); + return timeSinceLastAttempt > timeUntilNextAttempt; + } + /** * Get the Resourece Data from Node through DMI Passthrough service. * diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncSpec.groovy index 614783efd..0f89a428b 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/ModuleSyncSpec.groovy @@ -90,16 +90,25 @@ class ModuleSyncSpec extends Specification { } - def 'Schedule a Cm-Handle Sync for LOCKED with reason LOCKED_MISBEHAVING Cm-Handles '() { + def 'Schedule a Cm-Handle Sync for LOCKED with reason LOCKED_MISBEHAVING Cm-Handles with #scenario'() { given: 'cm handles in an locked state' def compositeState = new CompositeStateBuilder().withCmHandleState(CmHandleState.LOCKED) - .withLockReason(LockReasonCategory.LOCKED_MISBEHAVING, '').build() + .withLockReason(LockReasonCategory.LOCKED_MISBEHAVING, '').withLastUpdatedTimeNow().build() def yangModelCmHandle = new YangModelCmHandle(id: 'some-cm-handle', compositeState: compositeState) and: 'sync utilities return a cm handle twice' mockSyncUtils.getLockedMisbehavingYangModelCmHandles() >> [yangModelCmHandle, yangModelCmHandle] + and: 'inventory persistence returns the composite state of the cm handle' + mockInventoryPersistence.getCmHandleState(yangModelCmHandle.getId()) >> compositeState + and: 'sync utils retry locked cm handle returns #isReadyForRetry' + mockSyncUtils.isReadyForRetry(compositeState) >>> isReadyForRetry when: 'module sync poll is executed' - objectUnderTest.executeLockedMisbehavingCmHandlePoll() + objectUnderTest.executeLockedCmHandlePoll() then: 'the first cm handle is updated to state "ADVISED" from "READY"' - 2 * mockInventoryPersistence.saveCmHandleState(yangModelCmHandle.id, compositeState) + expectedNumberOfInvocationsToSaveCmHandleState * mockInventoryPersistence.saveCmHandleState(yangModelCmHandle.id, compositeState) + where: + scenario | isReadyForRetry || expectedNumberOfInvocationsToSaveCmHandleState + 'retry locked cm handle once' | [true, false] || 1 + 'retry locked cm handle twice' | [true, true] || 2 + 'do not retry locked cm handle' | [false, false] || 0 } } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/SyncUtilsSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/SyncUtilsSpec.groovy index 2c45ab769..dd299146f 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/SyncUtilsSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/sync/SyncUtilsSpec.groovy @@ -27,6 +27,7 @@ import org.onap.cps.ncmp.api.impl.operations.DmiDataOperations import org.onap.cps.ncmp.api.impl.operations.DmiOperations import org.onap.cps.ncmp.api.inventory.CmHandleState import org.onap.cps.ncmp.api.inventory.CompositeState +import org.onap.cps.ncmp.api.inventory.CompositeStateBuilder import org.onap.cps.ncmp.api.inventory.InventoryPersistence import org.onap.cps.ncmp.api.inventory.LockReasonCategory import org.onap.cps.ncmp.api.inventory.SyncState @@ -38,6 +39,9 @@ import org.springframework.http.ResponseEntity import spock.lang.Shared import spock.lang.Specification +import java.time.OffsetDateTime +import java.time.format.DateTimeFormatter + class SyncUtilsSpec extends Specification{ def mockInventoryPersistence = Mock(InventoryPersistence) @@ -48,6 +52,9 @@ class SyncUtilsSpec extends Specification{ def objectUnderTest = new SyncUtils(mockInventoryPersistence, mockDmiDataOperations, jsonObjectMapper) + @Shared + def formattedDateAndTime = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ").format(OffsetDateTime.now()) + @Shared def dataNode = new DataNode(leaves: ['id': 'cm-handle-123']) @@ -94,6 +101,21 @@ class SyncUtilsSpec extends Specification{ result[0].id == 'cm-handle-123' } + def 'Retry Locked Cm-Handle where the last update time is #scenario'() { + when: 'retry locked cm handle is invoked' + def result = objectUnderTest.isReadyForRetry(new CompositeStateBuilder() + .withLockReason(LockReasonCategory.LOCKED_MISBEHAVING, details) + .withLastUpdatedTime(lastUpdateTime).build()) + then: 'result returns #expectedResult' + result == expectedResult + where: + scenario | lastUpdateTime | details || expectedResult + 'is the first attempt' | '1900-01-01T00:00:00.000+0100' | 'First Attempt' || true + 'is greater than one minute' | '1900-01-01T00:00:00.000+0100' | 'Attempt #1 failed:' || true + 'is less than eight minutes' | formattedDateAndTime | 'Attempt #3 failed:' || false + } + + def 'Get a Cm-Handle where Operational Sync state is UnSynchronized and Cm-handle state is READY and #scenario'() { given: 'the inventory persistence service returns a collection of data nodes' mockInventoryPersistence.getCmHandlesByOperationalSyncState(SyncState.UNSYNCHRONIZED) >> unSynchronizedDataNodes -- 2.16.6