From 60dc411a89684d25b573ffb19b9759000cabdceb Mon Sep 17 00:00:00 2001 From: danielhanrahan Date: Tue, 20 Feb 2024 14:50:14 +0000 Subject: [PATCH] Test of retry of failed module sync A new integration test verifies that LOCKED handles that failed module sync will be retried after a few minutes. Issue-ID: CPS-2033 Signed-off-by: danielhanrahan Change-Id: If194daeb315090f2710a017270875b6301f7140a --- .../integration/base/CpsIntegrationSpecBase.groovy | 36 ++++++++++---- .../functional/NcmpCmHandleCreateSpec.groovy | 58 +++++++++++++++++++--- .../functional/NcmpCmHandleUpgradeSpec.groovy | 2 +- 3 files changed, 78 insertions(+), 18 deletions(-) diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy index 23504e49c..21e225e0b 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/base/CpsIntegrationSpecBase.groovy @@ -53,8 +53,13 @@ import spock.lang.Shared import spock.lang.Specification import spock.util.concurrent.PollingConditions +import java.time.format.DateTimeFormatter + import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus +import static org.onap.cps.ncmp.api.impl.ncmppersistence.NcmpPersistence.NCMP_DATASPACE_NAME; +import static org.onap.cps.ncmp.api.impl.ncmppersistence.NcmpPersistence.NCMP_DMI_REGISTRY_ANCHOR; +import static org.onap.cps.ncmp.api.impl.ncmppersistence.NcmpPersistence.NCMP_DMI_REGISTRY_PARENT; @SpringBootTest(classes = [CpsDataspaceService]) @Testcontainers @@ -116,7 +121,7 @@ abstract class CpsIntegrationSpecBase extends Specification { createStandardBookStoreSchemaSet(GENERAL_TEST_DATASPACE) initialized = true } - mockDmiServer = MockRestServiceServer.createServer(restTemplate) + mockDmiServer = MockRestServiceServer.bindTo(restTemplate).ignoreExpectOrder(true).build() } def cleanup() { @@ -188,7 +193,7 @@ abstract class CpsIntegrationSpecBase extends Specification { def registerCmHandle(dmiPlugin, cmHandleId, moduleSetTag, dmiModuleReferencesResponse, dmiModuleResourcesResponse) { def cmHandleToCreate = new NcmpServiceCmHandle(cmHandleId: cmHandleId, moduleSetTag: moduleSetTag) networkCmProxyDataService.updateDmiRegistrationAndSyncModule(new DmiPluginRegistration(dmiPlugin: dmiPlugin, createdCmHandles: [cmHandleToCreate])) - mockDmiResponsesForRegistration(dmiPlugin, cmHandleId, dmiModuleReferencesResponse, dmiModuleResourcesResponse) + mockDmiResponsesForModuleSync(dmiPlugin, cmHandleId, dmiModuleReferencesResponse, dmiModuleResourcesResponse) moduleSyncWatchdog.moduleSyncAdvisedCmHandles() new PollingConditions().within(3, () -> { CmHandleState.READY == networkCmProxyDataService.getCmHandleCompositeState(cmHandleId).cmHandleState @@ -204,14 +209,23 @@ abstract class CpsIntegrationSpecBase extends Specification { networkCmProxyDataService.updateDmiRegistrationAndSyncModule(new DmiPluginRegistration(dmiPlugin: dmiPlugin, removedCmHandles: cmHandleIds)) } - def mockDmiResponsesForRegistration(dmiPlugin, cmHandleId, dmiModuleReferencesResponse, dmiModuleResourcesResponse) { - if (dmiModuleReferencesResponse != null) { - mockDmiServer.expect(requestTo("${dmiPlugin}/dmi/v1/ch/${cmHandleId}/modules")) - .andRespond(withStatus(HttpStatus.OK).contentType(MediaType.APPLICATION_JSON).body(dmiModuleReferencesResponse)) - } - if (dmiModuleResourcesResponse != null) { - mockDmiServer.expect(requestTo("${dmiPlugin}/dmi/v1/ch/${cmHandleId}/moduleResources")) - .andRespond(withStatus(HttpStatus.OK).contentType(MediaType.APPLICATION_JSON).body(dmiModuleResourcesResponse)) - } + def mockDmiResponsesForModuleSync(dmiPlugin, cmHandleId, dmiModuleReferencesResponse, dmiModuleResourcesResponse) { + mockDmiServer.expect(requestTo("${dmiPlugin}/dmi/v1/ch/${cmHandleId}/modules")) + .andRespond(withStatus(HttpStatus.OK).contentType(MediaType.APPLICATION_JSON).body(dmiModuleReferencesResponse)) + mockDmiServer.expect(requestTo("${dmiPlugin}/dmi/v1/ch/${cmHandleId}/moduleResources")) + .andRespond(withStatus(HttpStatus.OK).contentType(MediaType.APPLICATION_JSON).body(dmiModuleResourcesResponse)) + } + + def mockDmiIsNotAvailableForModuleSync(dmiPlugin, cmHandleId) { + mockDmiServer.expect(requestTo("${dmiPlugin}/dmi/v1/ch/${cmHandleId}/modules")) + .andRespond(withStatus(HttpStatus.SERVICE_UNAVAILABLE)) + } + + def overrideCmHandleLastUpdateTime(cmHandleId, newUpdateTime) { + String ISO_TIMESTAMP_PATTERN = "yyyy-MM-dd'T'HH:mm:ss.SSSZ"; + DateTimeFormatter ISO_TIMESTAMP_FORMATTER = DateTimeFormatter.ofPattern(ISO_TIMESTAMP_PATTERN); + def jsonForUpdate = '{ "state": { "last-update-time": "%s" } }'.formatted(ISO_TIMESTAMP_FORMATTER.format(newUpdateTime)) + cpsDataService.updateNodeLeaves(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR, + NCMP_DMI_REGISTRY_PARENT + "/cm-handles[@id='${cmHandleId}']", jsonForUpdate, now) } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/NcmpCmHandleCreateSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/NcmpCmHandleCreateSpec.groovy index 4369b799b..6b6f62edf 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/NcmpCmHandleCreateSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/NcmpCmHandleCreateSpec.groovy @@ -20,6 +20,7 @@ package org.onap.cps.integration.functional +import java.time.OffsetDateTime import org.onap.cps.integration.base.CpsIntegrationSpecBase import org.onap.cps.ncmp.api.NetworkCmProxyDataService import org.onap.cps.ncmp.api.impl.inventory.CmHandleState @@ -27,12 +28,8 @@ import org.onap.cps.ncmp.api.impl.inventory.LockReasonCategory import org.onap.cps.ncmp.api.models.CmHandleRegistrationResponse import org.onap.cps.ncmp.api.models.DmiPluginRegistration import org.onap.cps.ncmp.api.models.NcmpServiceCmHandle -import org.springframework.http.HttpStatus import spock.util.concurrent.PollingConditions -import static org.springframework.test.web.client.match.MockRestRequestMatchers.anything -import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus - class NcmpCmHandleCreateSpec extends CpsIntegrationSpecBase { NetworkCmProxyDataService objectUnderTest @@ -48,7 +45,7 @@ class NcmpCmHandleCreateSpec extends CpsIntegrationSpecBase { def 'CM Handle registration is successful.'() { given: 'DMI will return modules when requested' - mockDmiResponsesForRegistration(DMI_URL, 'ch-1', MODULE_REFERENCES_RESPONSE_A, MODULE_RESOURCES_RESPONSE_A) + mockDmiResponsesForModuleSync(DMI_URL, 'ch-1', MODULE_REFERENCES_RESPONSE_A, MODULE_RESOURCES_RESPONSE_A) when: 'a CM-handle is registered for creation' def cmHandleToCreate = new NcmpServiceCmHandle(cmHandleId: 'ch-1') @@ -81,7 +78,7 @@ class NcmpCmHandleCreateSpec extends CpsIntegrationSpecBase { def 'CM Handle goes to LOCKED state when DMI gives error during module sync.'() { given: 'DMI is not available to handle requests' - mockDmiServer.expect(anything()).andRespond(withStatus(HttpStatus.SERVICE_UNAVAILABLE)) + mockDmiIsNotAvailableForModuleSync(DMI_URL, 'ch-1') when: 'a CM-handle is registered for creation' def cmHandleToCreate = new NcmpServiceCmHandle(cmHandleId: 'ch-1') @@ -131,4 +128,53 @@ class NcmpCmHandleCreateSpec extends CpsIntegrationSpecBase { cleanup: 'deregister CM handles' deregisterCmHandles(DMI_URL, ['ch-1', 'ch-2', 'ch-3']) } + + def 'CM Handle retry after failed module sync.'() { + given: 'DMI is not initially available to handle requests' + mockDmiIsNotAvailableForModuleSync(DMI_URL, 'ch-1') + mockDmiIsNotAvailableForModuleSync(DMI_URL, 'ch-2') + and: 'DMI will be available for retry' + mockDmiResponsesForModuleSync(DMI_URL, 'ch-1', MODULE_REFERENCES_RESPONSE_A, MODULE_RESOURCES_RESPONSE_A) + mockDmiResponsesForModuleSync(DMI_URL, 'ch-2', MODULE_REFERENCES_RESPONSE_B, MODULE_RESOURCES_RESPONSE_B) + + when: 'CM-handles are registered for creation' + def cmHandlesToCreate = [new NcmpServiceCmHandle(cmHandleId: 'ch-1'), new NcmpServiceCmHandle(cmHandleId: 'ch-2')] + def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: DMI_URL, createdCmHandles: cmHandlesToCreate) + networkCmProxyDataService.updateDmiRegistrationAndSyncModule(dmiPluginRegistration) + and: 'module sync runs' + moduleSyncWatchdog.moduleSyncAdvisedCmHandles() + then: 'CM-handles go to LOCKED state' + new PollingConditions().within(3, () -> { + assert objectUnderTest.getCmHandleCompositeState('ch-1').cmHandleState == CmHandleState.LOCKED + assert objectUnderTest.getCmHandleCompositeState('ch-2').cmHandleState == CmHandleState.LOCKED + }) + + when: 'we wait for LOCKED CM handle retry time (actually just subtract 3 minutes from handles lastUpdateTime)' + overrideCmHandleLastUpdateTime('ch-1', OffsetDateTime.now().minusMinutes(3)) + overrideCmHandleLastUpdateTime('ch-2', OffsetDateTime.now().minusMinutes(3)) + and: 'failed CM handles are reset' + moduleSyncWatchdog.resetPreviouslyFailedCmHandles() + then: 'CM-handles are ADVISED state' + assert objectUnderTest.getCmHandleCompositeState('ch-1').cmHandleState == CmHandleState.ADVISED + assert objectUnderTest.getCmHandleCompositeState('ch-2').cmHandleState == CmHandleState.ADVISED + + when: 'module sync runs' + moduleSyncWatchdog.moduleSyncAdvisedCmHandles() + then: 'CM-handles go to READY state' + new PollingConditions().within(3, () -> { + assert objectUnderTest.getCmHandleCompositeState('ch-1').cmHandleState == CmHandleState.READY + assert objectUnderTest.getCmHandleCompositeState('ch-2').cmHandleState == CmHandleState.READY + }) + and: 'CM-handles have expected modules' + assert ['M1', 'M2'] == objectUnderTest.getYangResourcesModuleReferences('ch-1').moduleName.sort() + assert ['M1', 'M3'] == objectUnderTest.getYangResourcesModuleReferences('ch-2').moduleName.sort() + and: 'CM-handles have expected module set tags (blank)' + assert objectUnderTest.getNcmpServiceCmHandle('ch-1').moduleSetTag == '' + assert objectUnderTest.getNcmpServiceCmHandle('ch-2').moduleSetTag == '' + and: 'DMI received expected requests' + mockDmiServer.verify() + + cleanup: 'deregister CM handle' + deregisterCmHandles(DMI_URL, ['ch-1', 'ch-2']) + } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/functional/NcmpCmHandleUpgradeSpec.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/functional/NcmpCmHandleUpgradeSpec.groovy index ffa4382d4..c5c59e05c 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/functional/NcmpCmHandleUpgradeSpec.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/functional/NcmpCmHandleUpgradeSpec.groovy @@ -55,7 +55,7 @@ class NcmpCmHandleUpgradeSpec extends CpsIntegrationSpecBase { assert ['M1', 'M2'] == objectUnderTest.getYangResourcesModuleReferences(CM_HANDLE_ID).moduleName.sort() and: 'DMI returns different modules for upgrade' - mockDmiResponsesForRegistration(DMI_URL, CM_HANDLE_ID, UPDATED_MODULE_REFERENCES_RESPONSE, UPDATED_MODULE_RESOURCES_RESPONSE) + mockDmiResponsesForModuleSync(DMI_URL, CM_HANDLE_ID, UPDATED_MODULE_REFERENCES_RESPONSE, UPDATED_MODULE_RESOURCES_RESPONSE) when: "CM-handle is upgraded with given moduleSetTag '${updatedModuleSetTag}'" def cmHandlesToUpgrade = new UpgradedCmHandles(cmHandles: [CM_HANDLE_ID], moduleSetTag: updatedModuleSetTag) -- 2.16.6