def compositeStateTestObject() {
new CompositeState(cmHandleState: CmHandleState.ADVISED,
- lockReason: CompositeState.LockReason.builder().lockReasonCategory(LockReasonCategory.LOCKED_MODULE_SYNC_FAILED).details("lock details").build(),
+ lockReason: CompositeState.LockReason.builder().lockReasonCategory(LockReasonCategory.MODULE_SYNC_FAILED).details("lock details").build(),
lastUpdateTime: formattedDateAndTime.toString(),
dataSyncEnabled: false,
dataStores: dataStores())
def compositeState = new CompositeStateBuilder()
.withCmHandleState(CmHandleState.ADVISED)
.withLastUpdatedTime(formattedDateAndTime.toString())
- .withLockReason(LockReasonCategory.LOCKED_MODULE_SYNC_FAILED, 'locked details')
+ .withLockReason(LockReasonCategory.MODULE_SYNC_FAILED, 'locked details')
.withOperationalDataStores(DataStoreSyncState.SYNCHRONIZED, formattedDateAndTime).build()
compositeState.setDataSyncEnabled(false)
when: 'mapper is called'
then: 'the composite state contains the expected lock Reason and details'
result.getLockReason().getReason() == expectedExternalLockReason
where:
- scenario | lockReason || expectedExternalLockReason
- 'LOCKED_MODULE_SYNC_FAILED' | LockReasonCategory.LOCKED_MODULE_SYNC_FAILED || 'LOCKED_MISBEHAVING'
- 'null value' | null || null
+ scenario | lockReason || expectedExternalLockReason
+ 'MODULE_SYNC_FAILED' | LockReasonCategory.MODULE_SYNC_FAILED || 'LOCKED_MISBEHAVING'
+ 'null value' | null || null
}
}
package org.onap.cps.ncmp.api.inventory;
public enum LockReasonCategory {
- LOCKED_MODULE_SYNC_FAILED
+ MODULE_SYNC_FAILED, MODULE_UPGRADE, MODULE_UPGRADE_FAILED
}
} catch (final Exception e) {
log.warn("Processing of {} module sync failed due to reason {}.", cmHandleId, e.getMessage());
syncUtils.updateLockReasonDetailsAndAttempts(compositeState,
- LockReasonCategory.LOCKED_MODULE_SYNC_FAILED, e.getMessage());
+ LockReasonCategory.MODULE_SYNC_FAILED, e.getMessage());
setCmHandleStateLocked(yangModelCmHandle, compositeState.getLockReason());
cmHandelStatePerCmHandle.put(yangModelCmHandle, CmHandleState.LOCKED);
}
final Map<YangModelCmHandle, CmHandleState> cmHandleStatePerCmHandle = new HashMap<>(failedCmHandles.size());
for (final YangModelCmHandle failedCmHandle : failedCmHandles) {
final CompositeState compositeState = failedCmHandle.getCompositeState();
- final boolean isReadyForRetry = syncUtils.isReadyForRetry(compositeState);
+ final boolean isReadyForRetry = syncUtils.needsModuleSyncRetry(compositeState);
log.info("Retry for cmHandleId : {} is {}", failedCmHandle.getId(), isReadyForRetry);
if (isReadyForRetry) {
final String resetCmHandleId = failedCmHandle.getId();
}
/**
- * Query data nodes for cm handles with an "LOCKED" cm handle state with reason LOCKED_MODULE_SYNC_FAILED".
+ * Query data nodes for cm handles with an "LOCKED" cm handle state with reason MODULE_SYNC_FAILED".
*
* @return a random LOCKED yang model cm handle, return null if not found
*/
public List<YangModelCmHandle> getModuleSyncFailedCmHandles() {
final List<DataNode> lockedCmHandlesAsDataNodeList = cmHandleQueries.queryCmHandleDataNodesByCpsPath(
- "//lock-reason[@reason=\"LOCKED_MODULE_SYNC_FAILED\"]",
+ "//lock-reason[@reason=\"MODULE_SYNC_FAILED\"]",
FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS);
return convertCmHandlesDataNodesToYangModelCmHandles(lockedCmHandlesAsDataNodeList);
}
/**
- * Check if the retry mechanism should attempt to unlock the cm handle based on the last update time.
+ * Check if a module sync retry is needed.
*
* @param compositeState the composite state currently in the locked state
* @return if the retry mechanism should be attempted
*/
- public boolean isReadyForRetry(final CompositeState compositeState) {
- int timeInMinutesUntilNextAttempt = 1;
+ public boolean needsModuleSyncRetry(final CompositeState compositeState) {
final OffsetDateTime time =
OffsetDateTime.parse(compositeState.getLastUpdateTime(),
DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ"));
final Matcher matcher = retryAttemptPattern.matcher(compositeState.getLockReason().getDetails());
+ final boolean failedDuringModuleSync = LockReasonCategory.MODULE_SYNC_FAILED
+ == compositeState.getLockReason().getLockReasonCategory();
+ if (!failedDuringModuleSync) {
+ log.info("Locked for other reason");
+ return false;
+ }
+ final int timeInMinutesUntilNextAttempt;
if (matcher.find()) {
timeInMinutesUntilNextAttempt = (int) Math.pow(2, Integer.parseInt(matcher.group(1)));
} else {
- log.debug("First Attempt: no current attempts found.");
+ timeInMinutesUntilNextAttempt = 1;
+ log.info("First Attempt: no current attempts found.");
}
final int timeSinceLastAttempt = (int) Duration.between(time, OffsetDateTime.now()).toMinutes();
if (timeInMinutesUntilNextAttempt >= timeSinceLastAttempt) {
log.info("Time until next attempt is {} minutes: ",
- timeInMinutesUntilNextAttempt - timeSinceLastAttempt);
+ timeInMinutesUntilNextAttempt - timeSinceLastAttempt);
+ return false;
}
- return timeSinceLastAttempt > timeInMinutesUntilNextAttempt;
+ log.info("Retry due now");
+ return true;
}
/**
given: 'the system returns a yang modelled cm handle'
def dmiServiceName = 'some service name'
def compositeState = new CompositeState(cmHandleState: CmHandleState.ADVISED,
- lockReason: CompositeState.LockReason.builder().lockReasonCategory(LockReasonCategory.LOCKED_MODULE_SYNC_FAILED).details("lock details").build(),
+ lockReason: CompositeState.LockReason.builder().lockReasonCategory(LockReasonCategory.MODULE_SYNC_FAILED).details("lock details").build(),
lastUpdateTime: 'some-timestamp',
dataSyncEnabled: false,
dataStores: dataStores())
def 'Get cm handle composite state'() {
given: 'a yang modelled cm handle'
def compositeState = new CompositeState(cmHandleState: CmHandleState.ADVISED,
- lockReason: CompositeState.LockReason.builder().lockReasonCategory(LockReasonCategory.LOCKED_MODULE_SYNC_FAILED).details("lock details").build(),
+ lockReason: CompositeState.LockReason.builder().lockReasonCategory(LockReasonCategory.MODULE_SYNC_FAILED).details("lock details").build(),
lastUpdateTime: 'some-timestamp',
dataSyncEnabled: false,
dataStores: dataStores())
import static org.onap.cps.ncmp.api.inventory.CmHandleState.DELETING
import static org.onap.cps.ncmp.api.inventory.CmHandleState.LOCKED
import static org.onap.cps.ncmp.api.inventory.CmHandleState.READY
-import static org.onap.cps.ncmp.api.inventory.LockReasonCategory.LOCKED_MODULE_SYNC_FAILED
+import static org.onap.cps.ncmp.api.inventory.LockReasonCategory.MODULE_SYNC_FAILED
class LcmEventsCmHandleStateHandlerImplSpec extends Specification {
def 'Update and Publish Events on State Change from LOCKED to ADVISED'() {
given: 'Cm Handle represented as YangModelCmHandle in LOCKED state'
compositeState = new CompositeState(cmHandleState: LOCKED,
- lockReason: CompositeState.LockReason.builder().lockReasonCategory(LOCKED_MODULE_SYNC_FAILED).details('some lock details').build())
+ lockReason: CompositeState.LockReason.builder().lockReasonCategory(MODULE_SYNC_FAILED).details('some lock details').build())
yangModelCmHandle = new YangModelCmHandle(id: cmHandleId, dmiProperties: [], publicProperties: [], compositeState: compositeState)
when: 'update state is invoked'
objectUnderTest.updateCmHandleState(yangModelCmHandle, ADVISED)
def compositeState = new CompositeStateBuilder()
.withCmHandleState(CmHandleState.LOCKED)
.withLastUpdatedTime('some-update-time')
- .withLockReason(LockReasonCategory.LOCKED_MODULE_SYNC_FAILED, 'locked details')
+ .withLockReason(LockReasonCategory.MODULE_SYNC_FAILED, 'locked details')
.withOperationalDataStores(DataStoreSyncState.SYNCHRONIZED, 'some-sync-time').build()
ncmpServiceCmHandle.setCompositeState(compositeState)
when: 'it is converted to a yang model cm handle'
import org.onap.cps.spi.model.DataNode
import org.onap.cps.spi.model.DataNodeBuilder
-import org.onap.cps.ncmp.api.inventory.CompositeStateBuilder
import spock.lang.Specification
import java.time.OffsetDateTime
def static cmHandleId = 'myHandle1'
def static cmHandleXpath = "/dmi-registry/cm-handles[@id='${cmHandleId}/state']"
def static stateDataNodes = [new DataNodeBuilder().withXpath("/dmi-registry/cm-handles[@id='${cmHandleId}']/state/lock-reason")
- .withLeaves(['reason': 'LOCKED_MODULE_SYNC_FAILED', 'details': 'lock details']).build(),
+ .withLeaves(['reason': 'MODULE_SYNC_FAILED', 'details': 'lock details']).build(),
new DataNodeBuilder().withXpath("/dmi-registry/cm-handles[@id='${cmHandleId}']/state/datastores")
.withChildDataNodes(Arrays.asList(new DataNodeBuilder()
.withXpath("/dmi-registry/cm-handles[@id='${cmHandleId}']/state/datastores/operational")
def "Composite State Specification"() {
when: 'using composite state builder '
def compositeState = new CompositeStateBuilder().withCmHandleState(CmHandleState.ADVISED)
- .withLockReason(LockReasonCategory.LOCKED_MODULE_SYNC_FAILED,"").withOperationalDataStores(DataStoreSyncState.UNSYNCHRONIZED,
+ .withLockReason(LockReasonCategory.MODULE_SYNC_FAILED,"").withOperationalDataStores(DataStoreSyncState.UNSYNCHRONIZED,
formattedDateAndTime.toString()).withLastUpdatedTime(formattedDateAndTime).build()
then: 'it matches expected cm handle state and data store sync state'
assert compositeState.cmHandleState == CmHandleState.ADVISED
def finalCompositeStateBuilder = new CompositeStateBuilder()
.withCmHandleState(CmHandleState.ADVISED)
.withLastUpdatedTime(formattedDateAndTime.toString())
- .withLockReason(LockReasonCategory.LOCKED_MODULE_SYNC_FAILED, 'locked details')
+ .withLockReason(LockReasonCategory.MODULE_SYNC_FAILED, 'locked details')
.withOperationalDataStores(DataStoreSyncState.SYNCHRONIZED, formattedDateAndTime)
when: 'build is called'
def result = finalCompositeStateBuilder.build()
and: 'built result should have correct values'
assert !result.getDataSyncEnabled()
assert result.getLastUpdateTime() == formattedDateAndTime
- assert result.getLockReason().getLockReasonCategory() == LockReasonCategory.LOCKED_MODULE_SYNC_FAILED
+ assert result.getLockReason().getLockReasonCategory() == LockReasonCategory.MODULE_SYNC_FAILED
assert result.getLockReason().getDetails() == 'locked details'
assert result.getCmHandleState() == CmHandleState.ADVISED
assert result.getDataStores().getOperationalDataStore().getDataStoreSyncState() == DataStoreSyncState.SYNCHRONIZED
def "Composite State Specification"() {
given: "a Composite State"
def compositeState = new CompositeState(cmHandleState: CmHandleState.ADVISED,
- lockReason: CompositeState.LockReason.builder().lockReasonCategory(LockReasonCategory.LOCKED_MODULE_SYNC_FAILED).details("lock details").build(),
+ lockReason: CompositeState.LockReason.builder().lockReasonCategory(LockReasonCategory.MODULE_SYNC_FAILED).details("lock details").build(),
lastUpdateTime: formattedDateAndTime.toString(),
dataSyncEnabled: false,
dataStores: dataStores())
when: 'module sync is executed'
objectUnderTest.performModuleSync([cmHandle], batchCount)
then: 'update lock reason, details and attempts is invoked'
- 1 * mockSyncUtils.updateLockReasonDetailsAndAttempts(cmHandleState, LockReasonCategory.LOCKED_MODULE_SYNC_FAILED, 'some exception')
+ 1 * mockSyncUtils.updateLockReasonDetailsAndAttempts(cmHandleState, LockReasonCategory.MODULE_SYNC_FAILED, 'some exception')
and: 'the state handler is called to update the state to LOCKED'
1 * mockLcmEventsCmHandleStateHandler.updateCmHandleStateBatch(_) >> { args ->
assertBatch(args, ['cm-handle'], CmHandleState.LOCKED)
def 'Reset failed CM Handles #scenario.'() {
given: 'cm handles in an locked state'
def lockedState = new CompositeStateBuilder().withCmHandleState(CmHandleState.LOCKED)
- .withLockReason(LockReasonCategory.LOCKED_MODULE_SYNC_FAILED, '').withLastUpdatedTimeNow().build()
+ .withLockReason(LockReasonCategory.MODULE_SYNC_FAILED, '').withLastUpdatedTimeNow().build()
def yangModelCmHandle1 = new YangModelCmHandle(id: 'cm-handle-1', compositeState: lockedState)
def yangModelCmHandle2 = new YangModelCmHandle(id: 'cm-handle-2', compositeState: lockedState)
def expectedCmHandleStatePerCmHandle = [(yangModelCmHandle1): CmHandleState.ADVISED]
moduleSyncStartedOnCmHandles.put('cm-handle-1', 'started')
moduleSyncStartedOnCmHandles.put('cm-handle-2', 'started')
and: 'sync utils retry locked cm handle returns #isReadyForRetry'
- mockSyncUtils.isReadyForRetry(lockedState) >>> isReadyForRetry
+ mockSyncUtils.needsModuleSyncRetry(lockedState) >>> isReadyForRetry
when: 'resetting failed cm handles'
objectUnderTest.resetFailedCmHandles([yangModelCmHandle1, yangModelCmHandle2])
then: 'updated to state "ADVISED" from "READY" is called as often as there are cm handles ready for retry'
package org.onap.cps.ncmp.api.inventory.sync
+import ch.qos.logback.classic.Level
+import ch.qos.logback.classic.Logger
+import ch.qos.logback.core.read.ListAppender
+import org.slf4j.LoggerFactory
+import org.springframework.context.annotation.AnnotationConfigApplicationContext
+
import static org.onap.cps.ncmp.api.impl.operations.DatastoreType.PASSTHROUGH_OPERATIONAL
+import static org.onap.cps.ncmp.api.inventory.LockReasonCategory.MODULE_SYNC_FAILED
+import static org.onap.cps.ncmp.api.inventory.LockReasonCategory.MODULE_UPGRADE
+import static org.onap.cps.ncmp.api.inventory.LockReasonCategory.MODULE_UPGRADE_FAILED
import com.fasterxml.jackson.databind.JsonNode
import com.fasterxml.jackson.databind.ObjectMapper
import org.onap.cps.ncmp.api.inventory.CompositeState
import org.onap.cps.ncmp.api.inventory.CompositeStateBuilder
import org.onap.cps.ncmp.api.inventory.DataStoreSyncState
-import org.onap.cps.ncmp.api.inventory.LockReasonCategory
import org.onap.cps.spi.FetchDescendantsOption
import org.onap.cps.spi.model.DataNode
import org.onap.cps.utils.JsonObjectMapper
import org.springframework.http.HttpStatus
import org.springframework.http.ResponseEntity
-import spock.lang.Shared
import spock.lang.Specification
import java.time.OffsetDateTime
import java.time.format.DateTimeFormatter
def objectUnderTest = new SyncUtils(mockCmHandleQueries, mockDmiDataOperations, jsonObjectMapper)
- @Shared
- def formattedDateAndTime = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ").format(OffsetDateTime.now())
+ def static neverUpdatedBefore = '1900-01-01T00:00:00.000+0100'
+
+ def static now = OffsetDateTime.now()
+
+ def static nowAsString = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ").format(now)
+
+ def static dataNode = new DataNode(leaves: ['id': 'cm-handle-123'])
+
+ def applicationContext = new AnnotationConfigApplicationContext()
- @Shared
- def dataNode = new DataNode(leaves: ['id': 'cm-handle-123'])
+ def logger = (Logger) LoggerFactory.getLogger(SyncUtils)
+ def loggingListAppender
+ void setup() {
+ logger.setLevel(Level.DEBUG)
+ loggingListAppender = new ListAppender()
+ logger.addAppender(loggingListAppender)
+ loggingListAppender.start()
+ applicationContext.refresh()
+ }
+
+ void cleanup() {
+ ((Logger) LoggerFactory.getLogger(SyncUtils.class)).detachAndStopAllAppenders()
+ applicationContext.close()
+ }
def 'Get an advised Cm-Handle where ADVISED cm handle #scenario'() {
given: 'the inventory persistence service returns a collection of data nodes'
given: 'A locked state'
def compositeState = new CompositeState(lockReason: lockReason)
when: 'update cm handle details and attempts is called'
- objectUnderTest.updateLockReasonDetailsAndAttempts(compositeState, LockReasonCategory.LOCKED_MODULE_SYNC_FAILED, 'new error message')
+ objectUnderTest.updateLockReasonDetailsAndAttempts(compositeState, MODULE_SYNC_FAILED, 'new error message')
then: 'the composite state lock reason and details are updated'
- assert compositeState.lockReason.lockReasonCategory == LockReasonCategory.LOCKED_MODULE_SYNC_FAILED
+ assert compositeState.lockReason.lockReasonCategory == MODULE_SYNC_FAILED
assert compositeState.lockReason.details == expectedDetails
where:
scenario | lockReason || expectedDetails
'exists' | CompositeState.LockReason.builder().details("Attempt #2 failed: some error message").build() || 'Attempt #3 failed: new error message'
}
- def 'Get all locked Cm-Handle where Lock Reason is LOCKED_MODULE_SYNC_FAILED cm handle #scenario'() {
+ def 'Get all locked Cm-Handle where Lock Reason is MODULE_SYNC_FAILED cm handle #scenario'() {
given: 'the cps (persistence service) returns a collection of data nodes'
mockCmHandleQueries.queryCmHandleDataNodesByCpsPath(
- '//lock-reason[@reason="LOCKED_MODULE_SYNC_FAILED"]',
+ '//lock-reason[@reason="MODULE_SYNC_FAILED"]',
FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS) >> [dataNode]
when: 'get locked Misbehaving cm handle is called'
def result = objectUnderTest.getModuleSyncFailedCmHandles()
}
def 'Retry Locked Cm-Handle where the last update time is #scenario'() {
- when: 'retry locked cm handle is invoked'
- def result = objectUnderTest.isReadyForRetry(new CompositeStateBuilder()
- .withLockReason(LockReasonCategory.LOCKED_MODULE_SYNC_FAILED, details)
- .withLastUpdatedTime(lastUpdateTime).build())
- then: 'result returns #expectedResult'
- result == expectedResult
- where:
- scenario | lastUpdateTime | details || expectedResult
- 'the first attempt' | '1900-01-01T00:00:00.000+0100' | 'First Attempt' || true
- 'greater than one minute' | '1900-01-01T00:00:00.000+0100' | 'Attempt #1 failed:' || true
- 'less than eight minutes' | formattedDateAndTime | 'Attempt #3 failed:' || false
+ given: 'Last update was #lastUpdateMinutesAgo minutes ago (-1 means never)'
+ def lastUpdatedTime = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ").format(now.minusMinutes(lastUpdateMinutesAgo))
+ if (lastUpdateMinutesAgo < 0 ) {
+ lastUpdatedTime = neverUpdatedBefore
+ }
+ when: 'checking to see if cm handle is ready for retry'
+ def result = objectUnderTest.needsModuleSyncRetry(new CompositeStateBuilder()
+ .withLockReason(MODULE_SYNC_FAILED, lockDetails)
+ .withLastUpdatedTime(lastUpdatedTime).build())
+ then: 'retry is only attempted when expected'
+ assert result == retryExpected
+ and: 'logs contain related information'
+ def logs = loggingListAppender.list.toString()
+ assert logs.contains(logReason)
+ where: 'the following parameters are used'
+ scenario | lastUpdateMinutesAgo | lockDetails | logReason || retryExpected
+ 'never attempted before' | -1 | 'fist attempt:' | 'First Attempt:' || true
+ '1st attempt, last attempt > 2 minute ago' | 3 | 'Attempt #1 failed:' | 'Retry due now' || true
+ '2nd attempt, last attempt < 4 minutes ago' | 1 | 'Attempt #2 failed:' | 'Time until next attempt is 3 minutes:' || false
+ '2nd attempt, last attempt > 4 minutes ago' | 5 | 'Attempt #2 failed:' | 'Retry due now' || true
}
+ def 'Retry Locked Cm-Handle with other lock reasons (category) #lockReasonCategory'() {
+ when: 'checking to see if cm handle is ready for retry'
+ def result = objectUnderTest.needsModuleSyncRetry(new CompositeStateBuilder()
+ .withLockReason(lockReasonCategory, 'some details')
+ .withLastUpdatedTime(nowAsString).build())
+ then: 'retry attempt is never triggered'
+ assert result == false
+ and: 'logs contain related information'
+ def logs = loggingListAppender.list.toString()
+ assert logs.contains('Locked for other reason')
+ where: 'the following lock reasons occurred'
+ lockReasonCategory << [MODULE_UPGRADE, MODULE_UPGRADE_FAILED]
+ }
def 'Get a Cm-Handle where #scenario'() {
given: 'the inventory persistence service returns a collection of data nodes'
{
"cm-handle-state" : "ADVISED",
"lock-reason" : {
- "reason" : "LOCKED_MODULE_SYNC_FAILED",
+ "reason" : "MODULE_SYNC_FAILED",
"details" : "lock details"
},
"last-update-time" : "2022-12-31T20:30:40.000+0000",
Code Review / cps.git / commitdiff