Adding retry logic for SDC Client in Distribution 01/68401/2
authorramverma <ram.krishna.verma@ericsson.com>
Fri, 21 Sep 2018 10:09:58 +0000 (11:09 +0100)
committerramverma <ram.krishna.verma@ericsson.com>
Fri, 21 Sep 2018 16:40:10 +0000 (17:40 +0100)
* SdcReceptionHandler in distribution tries to initialize & start the
SDC Client when distribution framework comes up. If by any chance the
SDC service is not up then currently the SdcReceptionHandler throws an
exception to the distribution activator and the distribution framework
goes down. And the health check fails.

* This review fixes it by adding a retry mechanism in
SdcReceptionHandler for all lifecycle methods of SdcClient (init, start,
stop). After failure the same operation is retried again with some
delay. The delay is passed as parameter from configuration json file.
The minimum default value for the delay is kept as 30 seconds.

* Adding SdcClientHandler timer task for performing the retries
asynchronously.

Change-Id: Ibb6d936fcf4872c82f87e2cd04a00583b81c92ff
Issue-ID: POLICY-1035
Signed-off-by: ramverma <ram.krishna.verma@ericsson.com>
plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcClientHandler.java [new file with mode: 0644]
plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandler.java
plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandlerConfigurationParameterBuilder.java
plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandlerConfigurationParameterGroup.java
plugins/reception-plugins/src/test/java/org/onap/policy/distribution/reception/handling/sdc/TestSdcReceptionHandler.java
plugins/reception-plugins/src/test/java/org/onap/policy/distribution/reception/handling/sdc/TestSdcReceptionHandlerConfigurationParameterGroup.java
plugins/reception-plugins/src/test/resources/handling-sdc.json

diff --git a/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcClientHandler.java b/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcClientHandler.java
new file mode 100644 (file)
index 0000000..f69fb15
--- /dev/null
@@ -0,0 +1,76 @@
+/*-
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2018 Ericsson. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.distribution.reception.handling.sdc;
+
+import java.util.Timer;
+import java.util.TimerTask;
+
+/**
+ * This class implements TimerTask for calling life cycle methods of SdcClient iteratively after specified interval
+ * until the operation is successful.
+ *
+ * @author Ram Krishna Verma (ram.krishna.verma@ericsson.com)
+ */
+public class SdcClientHandler extends TimerTask {
+
+    private SdcReceptionHandler sdcReceptionHandler;
+    private SdcClientOperationType operationType;
+    private Timer timer;
+
+    public enum SdcClientOperationType {
+        START, STOP
+    }
+
+    /**
+     * Constructs an instance of {@link SdcClientHandler} class.
+     *
+     * @param sdcReceptionHandler the sdcReceptionHandler
+     */
+    public SdcClientHandler(final SdcReceptionHandler sdcReceptionHandler, final SdcClientOperationType operationType,
+            final long retryDelay) {
+        this.sdcReceptionHandler = sdcReceptionHandler;
+        this.operationType = operationType;
+        timer = new Timer(false);
+        timer.scheduleAtFixedRate(this, 0, retryDelay * 1000L);
+    }
+
+    /**
+     * {@inheritDoc}.
+     */
+    @Override
+    public void run() {
+        if (SdcClientOperationType.START.equals(operationType)) {
+            sdcReceptionHandler.initializeSdcClient();
+            sdcReceptionHandler.startSdcClient();
+        } else {
+            sdcReceptionHandler.stopSdcClient();
+        }
+    }
+
+    /**
+     * {@inheritDoc}.
+     */
+    @Override
+    public boolean cancel() {
+        timer.cancel();
+        return true;
+    }
+}
index 443235b..2ada9fc 100644 (file)
@@ -30,10 +30,9 @@ import org.onap.policy.common.logging.flexlogger.FlexLogger;
 import org.onap.policy.common.logging.flexlogger.Logger;
 import org.onap.policy.common.parameters.ParameterService;
 import org.onap.policy.distribution.model.Csar;
-import org.onap.policy.distribution.reception.decoding.PluginInitializationException;
-import org.onap.policy.distribution.reception.decoding.PluginTerminationException;
 import org.onap.policy.distribution.reception.decoding.PolicyDecodingException;
 import org.onap.policy.distribution.reception.handling.AbstractReceptionHandler;
+import org.onap.policy.distribution.reception.handling.sdc.SdcClientHandler.SdcClientOperationType;
 import org.onap.policy.distribution.reception.handling.sdc.exceptions.ArtifactDownloadException;
 import org.onap.policy.distribution.reception.statistics.DistributionStatisticsManager;
 import org.onap.sdc.api.IDistributionClient;
@@ -54,38 +53,34 @@ import org.onap.sdc.utils.DistributionStatusEnum;
 public class SdcReceptionHandler extends AbstractReceptionHandler implements INotificationCallback {
 
     private static final Logger LOGGER = FlexLogger.getLogger(SdcReceptionHandler.class);
+    private static final String SECONDS = "Seconds";
 
     private SdcReceptionHandlerStatus sdcReceptionHandlerStatus = SdcReceptionHandlerStatus.STOPPED;
-    private SdcReceptionHandlerConfigurationParameterGroup handlerParameters;
     private IDistributionClient distributionClient;
     private SdcConfiguration sdcConfig;
     private volatile int nbOfNotificationsOngoing = 0;
+    private int retryDelay;
+    private SdcClientHandler sdcClientHandler;
 
     private enum DistributionStatusType {
         DOWNLOAD, DEPLOY
     }
 
     @Override
-    protected void initializeReception(final String parameterGroupName) throws PluginInitializationException {
-        handlerParameters = ParameterService.get(parameterGroupName);
-        initializeSdcClient();
-        startSdcClient();
+    protected void initializeReception(final String parameterGroupName) {
+        final SdcReceptionHandlerConfigurationParameterGroup handlerParameters =
+                ParameterService.get(parameterGroupName);
+        retryDelay = handlerParameters.getRetryDelay() < 30 ? 30 : handlerParameters.getRetryDelay();
+        sdcConfig = new SdcConfiguration(handlerParameters);
+        distributionClient = createSdcDistributionClient();
+        sdcClientHandler = new SdcClientHandler(this, SdcClientOperationType.START, retryDelay);
     }
 
     @Override
-    public void destroy() throws PluginTerminationException {
-        LOGGER.debug("Going to stop the SDC Client...");
+    public void destroy() {
         if (distributionClient != null) {
-            final IDistributionClientResult clientResult = distributionClient.stop();
-            if (!clientResult.getDistributionActionResult().equals(DistributionActionResultEnum.SUCCESS)) {
-                final String message =
-                        "SDC client stop failed with reason:" + clientResult.getDistributionMessageResult();
-                LOGGER.error(message);
-                throw new PluginTerminationException(message);
-            }
+            sdcClientHandler = new SdcClientHandler(this, SdcClientOperationType.STOP, retryDelay);
         }
-        changeSdcReceptionHandlerStatus(SdcReceptionHandlerStatus.STOPPED);
-        LOGGER.debug("SDC Client is stopped successfully");
     }
 
     @Override
@@ -115,6 +110,8 @@ public class SdcReceptionHandler extends AbstractReceptionHandler implements INo
                 ++nbOfNotificationsOngoing;
                 sdcReceptionHandlerStatus = newStatus;
                 break;
+            default:
+                break;
         }
     }
 
@@ -130,48 +127,61 @@ public class SdcReceptionHandler extends AbstractReceptionHandler implements INo
     /**
      * Method to initialize the SDC client.
      *
-     * @throws PluginInitializationException if the initialization of SDC Client fails
      */
-    private void initializeSdcClient() throws PluginInitializationException {
+    protected void initializeSdcClient() {
 
         LOGGER.debug("Initializing the SDC Client...");
         if (sdcReceptionHandlerStatus != SdcReceptionHandlerStatus.STOPPED) {
-            final String message = "The SDC Client is already initialized";
-            LOGGER.error(message);
-            throw new PluginInitializationException(message);
+            LOGGER.error("The SDC Client is already initialized");
+            return;
         }
-        sdcConfig = new SdcConfiguration(handlerParameters);
-        distributionClient = createSdcDistributionClient();
         final IDistributionClientResult clientResult = distributionClient.init(sdcConfig, this);
         if (!clientResult.getDistributionActionResult().equals(DistributionActionResultEnum.SUCCESS)) {
-            changeSdcReceptionHandlerStatus(SdcReceptionHandlerStatus.STOPPED);
-            final String message =
-                    "SDC client initialization failed with reason:" + clientResult.getDistributionMessageResult();
-            LOGGER.error(message);
-            throw new PluginInitializationException(message);
+            LOGGER.error("SDC client initialization failed with reason:" + clientResult.getDistributionMessageResult()
+                    + ". Initialization will be retried after " + retryDelay + SECONDS);
+            return;
         }
         LOGGER.debug("SDC Client is initialized successfully");
-        this.changeSdcReceptionHandlerStatus(SdcReceptionHandlerStatus.INIT);
+        changeSdcReceptionHandlerStatus(SdcReceptionHandlerStatus.INIT);
     }
 
     /**
      * Method to start the SDC client.
      *
-     * @param configParameter the configuration parameters
-     * @throws PluginInitializationException if the start of SDC Client fails
      */
-    private void startSdcClient() throws PluginInitializationException {
+    protected void startSdcClient() {
 
         LOGGER.debug("Going to start the SDC Client...");
+        if (sdcReceptionHandlerStatus != SdcReceptionHandlerStatus.INIT) {
+            LOGGER.error("The SDC Client is not initialized");
+            return;
+        }
         final IDistributionClientResult clientResult = distributionClient.start();
         if (!clientResult.getDistributionActionResult().equals(DistributionActionResultEnum.SUCCESS)) {
-            changeSdcReceptionHandlerStatus(SdcReceptionHandlerStatus.STOPPED);
-            final String message = "SDC client start failed with reason:" + clientResult.getDistributionMessageResult();
-            LOGGER.error(message);
-            throw new PluginInitializationException(message);
+            LOGGER.error("SDC client start failed with reason:" + clientResult.getDistributionMessageResult()
+                    + ". Start will be retried after " + retryDelay + SECONDS);
+            return;
         }
         LOGGER.debug("SDC Client is started successfully");
-        this.changeSdcReceptionHandlerStatus(SdcReceptionHandlerStatus.IDLE);
+        changeSdcReceptionHandlerStatus(SdcReceptionHandlerStatus.IDLE);
+        sdcClientHandler.cancel();
+    }
+
+    /**
+     * Method to stop the SDC client.
+     *
+     */
+    protected void stopSdcClient() {
+        LOGGER.debug("Going to stop the SDC Client...");
+        final IDistributionClientResult clientResult = distributionClient.stop();
+        if (!clientResult.getDistributionActionResult().equals(DistributionActionResultEnum.SUCCESS)) {
+            LOGGER.error("SDC client stop failed with reason:" + clientResult.getDistributionMessageResult()
+                    + ". Stop will be retried after " + retryDelay + SECONDS);
+            return;
+        }
+        LOGGER.debug("SDC Client is stopped successfully");
+        changeSdcReceptionHandlerStatus(SdcReceptionHandlerStatus.STOPPED);
+        sdcClientHandler.cancel();
     }
 
     /**
index 34eaf33..65305c1 100644 (file)
@@ -44,6 +44,7 @@ public class SdcReceptionHandlerConfigurationParameterBuilder {
     private String keystorePassword;
     private List<String> messageBusAddress;
     private List<String> artifactTypes;
+    private int retryDelay;
 
     /**
      * Set activeserverTlsAuth to this {@link SdcReceptionHandlerConfigurationParameterBuilder} instance.
@@ -196,6 +197,16 @@ public class SdcReceptionHandlerConfigurationParameterBuilder {
         return this;
     }
 
+    /**
+     * Set retryDelay to this {@link SdcReceptionHandlerConfigurationParameterBuilder} instance.
+     *
+     * @param retryDelay the retryDelay
+     */
+    public SdcReceptionHandlerConfigurationParameterBuilder setRetryDelay(final int retryDelay) {
+        this.retryDelay = retryDelay;
+        return this;
+    }
+
     /**
      * Returns the active server TlsAuth of this {@link SdcReceptionHandlerConfigurationParameterBuilder} instance.
      *
@@ -332,6 +343,15 @@ public class SdcReceptionHandlerConfigurationParameterBuilder {
         return artifactTypes;
     }
 
+    /**
+     * Returns the retryDelay of this {@link SdcReceptionHandlerConfigurationParameterBuilder} instance.
+     *
+     * @return the retryDelay
+     */
+    public int getRetryDelay() {
+        return retryDelay;
+    }
+
 }
 
 
index 0140fb5..bdc9767 100644 (file)
 package org.onap.policy.distribution.reception.handling.sdc;
 
 import java.util.List;
+
 import org.onap.policy.common.parameters.GroupValidationResult;
 import org.onap.policy.common.parameters.ValidationStatus;
 import org.onap.policy.common.utils.validation.ParameterValidationUtils;
 import org.onap.policy.distribution.reception.parameters.ReceptionHandlerConfigurationParameterGroup;
 
 /**
- * This class handles reading, parsing and validating of the Policy SDC Service Distribution
- * parameters from Json format, which strictly adheres to the interface:IConfiguration, defined by
- * SDC SDK.
+ * This class handles reading, parsing and validating of the Policy SDC Service Distribution parameters from Json
+ * format, which strictly adheres to the interface:IConfiguration, defined by SDC SDK.
  */
 public class SdcReceptionHandlerConfigurationParameterGroup extends ReceptionHandlerConfigurationParameterGroup {
 
@@ -39,6 +39,7 @@ public class SdcReceptionHandlerConfigurationParameterGroup extends ReceptionHan
     private String password;
     private int pollingInterval;
     private int pollingTimeout;
+    private int retryDelay;
     private String consumerId;
     private List<String> artifactTypes;
     private String consumerGroup;
@@ -50,8 +51,7 @@ public class SdcReceptionHandlerConfigurationParameterGroup extends ReceptionHan
     private boolean isUseHttpsWithDmaap;
 
     /**
-     * The constructor for instantiating {@link SdcReceptionHandlerConfigurationParameterGroup}
-     * class.
+     * The constructor for instantiating {@link SdcReceptionHandlerConfigurationParameterGroup} class.
      *
      * @param builder the SDC configuration builder
      */
@@ -63,6 +63,7 @@ public class SdcReceptionHandlerConfigurationParameterGroup extends ReceptionHan
         password = builder.getPassword();
         pollingInterval = builder.getPollingInterval();
         pollingTimeout = builder.getPollingTimeout();
+        retryDelay = builder.getRetryDelay();
         consumerId = builder.getConsumerId();
         artifactTypes = builder.getArtifactTypes();
         consumerGroup = builder.getConsumerGroup();
@@ -99,6 +100,10 @@ public class SdcReceptionHandlerConfigurationParameterGroup extends ReceptionHan
         return pollingTimeout;
     }
 
+    public int getRetryDelay() {
+        return retryDelay;
+    }
+
     public String getConsumerId() {
         return consumerId;
     }
@@ -149,6 +154,7 @@ public class SdcReceptionHandlerConfigurationParameterGroup extends ReceptionHan
         validateStringElement(validationResult, keyStorePassword, "keyStorePassword");
         validateIntElement(validationResult, pollingInterval, "pollingInterval");
         validateIntElement(validationResult, pollingTimeout, "pollingTimeout");
+        validateIntElement(validationResult, retryDelay, "retryDelay");
         validateStringListElement(validationResult, messageBusAddress, "messageBusAddress");
         validateStringListElement(validationResult, artifactTypes, "artifactTypes");
         return validationResult;
index bf42476..557a768 100644 (file)
@@ -51,7 +51,6 @@ import org.onap.policy.distribution.forwarding.PolicyForwarder;
 import org.onap.policy.distribution.forwarding.parameters.PolicyForwarderParameters;
 import org.onap.policy.distribution.model.Csar;
 import org.onap.policy.distribution.reception.decoding.PluginInitializationException;
-import org.onap.policy.distribution.reception.decoding.PluginTerminationException;
 import org.onap.policy.distribution.reception.decoding.PolicyDecoder;
 import org.onap.policy.distribution.reception.handling.AbstractReceptionHandler;
 import org.onap.policy.distribution.reception.handling.PluginHandler;
@@ -97,9 +96,14 @@ public class TestSdcReceptionHandler {
      * Setup for the test cases.
      *
      * @throws IOException if it occurs
+     * @throws SecurityException if it occurs
+     * @throws NoSuchFieldException if it occurs
+     * @throws IllegalAccessException if it occurs
+     * @throws IllegalArgumentException if it occurs
      */
     @Before
-    public final void init() throws IOException {
+    public final void init() throws IOException, NoSuchFieldException, SecurityException, IllegalArgumentException,
+            IllegalAccessException {
         DistributionStatisticsManager.resetAllStatistics();
         final Gson gson = new GsonBuilder().create();
         pssdConfigParameters = gson.fromJson(new FileReader("src/test/resources/handling-sdc.json"),
@@ -140,45 +144,34 @@ public class TestSdcReceptionHandler {
     public final void testInitializeSdcClient() {
         try {
             sypHandler.initializeReception(pssdConfigParameters.getName());
-        } catch (final PluginInitializationException exp) {
+        } catch (final Exception exp) {
             LOGGER.error(exp);
             fail("Test should not throw any exception");
         }
     }
 
     @Test
-    public final void testInitializeSdcClient_Again() throws PluginInitializationException {
-        sypHandler.initializeReception(pssdConfigParameters.getName());
-        try {
-            sypHandler.initializeReception(pssdConfigParameters.getName());
-            fail("Test must throw an exception here");
-        } catch (final Exception exp) {
-            assertTrue(exp.getMessage().startsWith("The SDC Client is already initialized"));
-        }
-    }
-
-    @Test
-    public final void testInitializeSdcClient_Failure() throws PluginInitializationException {
+    public final void testInitializeSdcClient_Failure() {
 
         Mockito.when(successfulClientInitResult.getDistributionActionResult())
-                .thenReturn(DistributionActionResultEnum.FAIL);
+                .thenReturn(DistributionActionResultEnum.FAIL).thenReturn(DistributionActionResultEnum.SUCCESS);
         try {
             sypHandler.initializeReception(pssdConfigParameters.getName());
-            fail("Test must throw an exception here");
         } catch (final Exception exp) {
-            assertTrue(exp.getMessage().startsWith("SDC client initialization failed with reason"));
+            LOGGER.error(exp);
+            fail("Test should not throw any exception");
         }
     }
 
     @Test
-    public final void testStartSdcClient_Failure() throws PluginInitializationException {
+    public final void testStartSdcClient_Failure() {
         try {
-            Mockito.when(distributionClient.start()).thenReturn(failureClientInitResult);
+            Mockito.when(distributionClient.start()).thenReturn(failureClientInitResult)
+                    .thenReturn(successfulClientInitResult);
             sypHandler.initializeReception(pssdConfigParameters.getName());
-
-            fail("Test must throw an exception here");
         } catch (final Exception exp) {
-            assertTrue(exp.getMessage().startsWith("SDC client start failed with reason"));
+            LOGGER.error(exp);
+            fail("Test should not throw any exception");
         }
     }
 
@@ -187,7 +180,7 @@ public class TestSdcReceptionHandler {
         try {
             sypHandler.initializeReception(pssdConfigParameters.getName());
             sypHandler.destroy();
-        } catch (final PluginInitializationException | PluginTerminationException exp) {
+        } catch (final Exception exp) {
             LOGGER.error(exp);
             fail("Test should not throw any exception");
         }
@@ -195,28 +188,28 @@ public class TestSdcReceptionHandler {
     }
 
     @Test
-    public final void testStopSdcClientWithoutStart() {
+    public final void testStopSdcClient_Failure() throws PluginInitializationException {
+
+        sypHandler.initializeReception(pssdConfigParameters.getName());
+        Mockito.when(distributionClient.stop()).thenReturn(failureClientInitResult)
+                .thenReturn(successfulClientInitResult);
         try {
             sypHandler.destroy();
-        } catch (final PluginTerminationException exp) {
+        } catch (final Exception exp) {
             LOGGER.error(exp);
             fail("Test should not throw any exception");
         }
-
     }
 
     @Test
-    public final void testStopSdcClient_Failure() throws PluginInitializationException {
-
-        sypHandler.initializeReception(pssdConfigParameters.getName());
-        Mockito.when(successfulClientInitResult.getDistributionActionResult())
-                .thenReturn(DistributionActionResultEnum.FAIL);
+    public final void testStopSdcClientWithoutStart() {
         try {
             sypHandler.destroy();
-            fail("Test must throw an exception here");
         } catch (final Exception exp) {
-            assertTrue(exp.getMessage().startsWith("SDC client stop failed with reason"));
+            LOGGER.error(exp);
+            fail("Test should not throw any exception");
         }
+
     }
 
     @Test
index df72d45..f9a65c5 100644 (file)
@@ -92,8 +92,8 @@ public class TestSdcReceptionHandlerConfigurationParameterGroup {
                 new SdcReceptionHandlerConfigurationParameterBuilder().setAsdcAddress("localhost")
                         .setConsumerGroup("policy-group").setConsumerId("policy-id").setEnvironmentName("TEST")
                         .setKeystorePassword("password").setKeystorePath("dummyPath").setPassword("policy")
-                        .setPollingInterval(10).setPollingTimeout(20).setUser("policy").setUseHttpsWithDmaap(false)
-                        .setActiveserverTlsAuth(false).setFilterinEmptyResources(true)
+                        .setPollingInterval(10).setPollingTimeout(20).setRetryDelay(30).setUser("policy")
+                        .setUseHttpsWithDmaap(false).setActiveserverTlsAuth(false).setFilterinEmptyResources(true)
                         .setArtifactTypes(Arrays.asList("TOSCA_CSAR")).setMessageBusAddress(Arrays.asList("localhost"));
         final SdcReceptionHandlerConfigurationParameterGroup configParameters =
                 new SdcReceptionHandlerConfigurationParameterGroup(builder);
@@ -106,6 +106,7 @@ public class TestSdcReceptionHandlerConfigurationParameterGroup {
         assertEquals("policy", configParameters.getPassword());
         assertEquals(10, configParameters.getPollingInterval());
         assertEquals(20, configParameters.getPollingTimeout());
+        assertEquals(30, configParameters.getRetryDelay());
         assertEquals("policy-id", configParameters.getConsumerId());
         assertEquals("policy-group", configParameters.getConsumerGroup());
         assertEquals("TEST", configParameters.getEnvironmentName());
index 7e3758d..0bb936b 100644 (file)
@@ -10,6 +10,7 @@
     "password": "policy",
     "pollingInterval":20,
     "pollingTimeout":30,
+    "retryDelay":30,
     "consumerId": "policy-id",
     "artifactTypes": [
         "TOSCA_CSAR",