[CCSDK-1921]Connection Timeout and Info Logging

[ccsdk/sli/plugins.git] / grToolkit / provider / src / main / java / org / onap / ccsdk / sli / plugins / grtoolkit / GrToolkitProvider.java

diff --git a/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/GrToolkitProvider.java b/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/GrToolkitProvider.java
index 4e121e3..116afb3 100755 (executable)
--- a/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/GrToolkitProvider.java
+++ b/grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/GrToolkitProvider.java
@@ -48,7 +48,6 @@ import javax.annotation.Nonnull;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
 
-import org.onap.ccsdk.sli.core.dblib.DBLibConnection;
 import org.onap.ccsdk.sli.core.dblib.DbLibService;
 import org.onap.ccsdk.sli.plugins.grtoolkit.data.ClusterActor;
 import org.onap.ccsdk.sli.plugins.grtoolkit.data.MemberBuilder;
@@ -103,6 +102,8 @@ public class GrToolkitProvider implements AutoCloseable, GrToolkitService, DataT
     private static final String HEALTHY = "HEALTHY";
     private static final String FAULTY = "FAULTY";
     private static final String VALUE = "value";
+    private static final String OUTPUT = "output";
+    private static final int CONNECTION_TIMEOUT = 5000; // 5 second timeout
     private String akkaConfig;
     private String jolokiaClusterPath;
     private String shardManagerPath;
@@ -244,6 +245,7 @@ public class GrToolkitProvider implements AutoCloseable, GrToolkitService, DataT
         DatabaseHealthOutputBuilder outputBuilder = new DatabaseHealthOutputBuilder();
         outputBuilder.setStatus("200");
         outputBuilder.setHealth(getDatabaseHealth());
+        outputBuilder.setServedBy(member);
 
         return Futures.immediateFuture(RpcResultBuilder.<DatabaseHealthOutput>status(true).withResult(outputBuilder.build()).build());
     }
@@ -254,7 +256,8 @@ public class GrToolkitProvider implements AutoCloseable, GrToolkitService, DataT
         AdminHealthOutputBuilder outputBuilder = new AdminHealthOutputBuilder();
         outputBuilder.setStatus("200");
         outputBuilder.setHealth(getAdminHealth());
-
+        outputBuilder.setServedBy(member);
+        log.info(outputBuilder.build().toString());
         return Futures.immediateFuture(RpcResultBuilder.<AdminHealthOutput>status(true).withResult(outputBuilder.build()).build());
     }
 
@@ -264,6 +267,7 @@ public class GrToolkitProvider implements AutoCloseable, GrToolkitService, DataT
         HaltAkkaTrafficOutputBuilder outputBuilder = new HaltAkkaTrafficOutputBuilder();
         outputBuilder.setStatus("200");
         modifyIpTables(IpTables.ADD, input.getNodeInfo().toArray());
+        outputBuilder.setServedBy(member);
 
         return Futures.immediateFuture(RpcResultBuilder.<HaltAkkaTrafficOutput>status(true).withResult(outputBuilder.build()).build());
     }
@@ -274,6 +278,7 @@ public class GrToolkitProvider implements AutoCloseable, GrToolkitService, DataT
         ResumeAkkaTrafficOutputBuilder outputBuilder = new ResumeAkkaTrafficOutputBuilder();
         outputBuilder.setStatus("200");
         modifyIpTables(IpTables.DELETE, input.getNodeInfo().toArray());
+        outputBuilder.setServedBy(member);
 
         return Futures.immediateFuture(RpcResultBuilder.<ResumeAkkaTrafficOutput>status(true).withResult(outputBuilder.build()).build());
     }
@@ -284,6 +289,7 @@ public class GrToolkitProvider implements AutoCloseable, GrToolkitService, DataT
         SiteIdentifierOutputBuilder outputBuilder = new SiteIdentifierOutputBuilder();
         outputBuilder.setStatus("200");
         outputBuilder.setId(siteIdentifier);
+        outputBuilder.setServedBy(member);
 
         return Futures.immediateFuture(RpcResultBuilder.<SiteIdentifierOutput>status(true).withResult(outputBuilder.build()).build());
     }
@@ -292,6 +298,7 @@ public class GrToolkitProvider implements AutoCloseable, GrToolkitService, DataT
     public ListenableFuture<RpcResult<FailoverOutput>> failover(FailoverInput input) {
         log.info("{}:failover invoked.", APP_NAME);
         FailoverOutputBuilder outputBuilder = new FailoverOutputBuilder();
+        outputBuilder.setServedBy(member);
         if(siteConfiguration != SiteConfiguration.GEO) {
             log.info("Cannot failover non-GEO site.");
             outputBuilder.setMessage("Failover aborted. This is not a GEO configuration.");
@@ -476,7 +483,12 @@ public class GrToolkitProvider implements AutoCloseable, GrToolkitService, DataT
             outputBuilder.setSite2Health(FAULTY);
         }
 
+        outputBuilder.setServedBy(member);
         RpcResult<ClusterHealthOutput> rpcResult = RpcResultBuilder.<ClusterHealthOutput>status(true).withResult(outputBuilder.build()).build();
+        log.info("{}:cluster-health: Site 1 | Healthy ODLs {}", APP_NAME, site1Health);
+        if(siteConfiguration == SiteConfiguration.GEO) {
+            log.info("{}:cluster-health: Site 2 | Healthy ODLs {}", APP_NAME, site2Health);
+        }
         return Futures.immediateFuture(rpcResult);
     }
 
@@ -533,12 +545,13 @@ public class GrToolkitProvider implements AutoCloseable, GrToolkitService, DataT
                             if(!performedCrossSiteHealthCheck) {
                                 try {
                                     String content = getRequestContent(httpProtocol + clusterActor.getNode() + ":" + port + "/restconf/operations/gr-toolkit:site-identifier", HttpMethod.POST);
-                                    crossSiteIdentifier = new JSONObject(content).getJSONObject("output").getString("id");
+                                    crossSiteIdentifier = new JSONObject(content).getJSONObject(OUTPUT).getString("id");
                                     crossSiteDbHealthy = crossSiteHealthRequest(httpProtocol + clusterActor.getNode() + ":" + port + "/restconf/operations/gr-toolkit:database-health");
                                     crossSiteAdminHealthy = crossSiteHealthRequest(httpProtocol + clusterActor.getNode() + ":" + port + "/restconf/operations/gr-toolkit:admin-health");
                                     performedCrossSiteHealthCheck = true;
                                 } catch(Exception e) {
-                                    log.info("Cannot get site identifier from {}", clusterActor.getNode());
+                                    log.info("Cannot get cross site health from {}", clusterActor.getNode());
+                                    log.info("siteIdentifier: {} | dbHealth: {} | adminHealth: {}", crossSiteIdentifier, crossSiteDbHealthy, crossSiteAdminHealthy);
                                     log.error("Site Health Error", e);
                                 }
                             }
@@ -549,6 +562,8 @@ public class GrToolkitProvider implements AutoCloseable, GrToolkitService, DataT
                 outputBuilder.getSites().add(builder.build());
                 builder = getSitesBuilder(site2HealthyODLs, site2Voting, crossSiteAdminHealthy, crossSiteDbHealthy, crossSiteIdentifier);
                 outputBuilder.getSites().add(builder.build());
+                log.info("{}:site-health: Site 1 ({}) | hasVotingMembers?: {} | Healthy ODLs: {} | ADM isHealthy?: {} | DB isHealthy?: {}", APP_NAME, siteIdentifier, site1Voting, site1HealthyODLs, HEALTHY.equals(adminHealth), HEALTHY.equals(databaseHealth));
+                log.info("{}:site-health: Site 2 ({}) | hasVotingMembers?: {} | Healthy ODLs: {} | ADM isHealthy?: {} | DB isHealthy?: {}", APP_NAME, crossSiteIdentifier, site2Voting, site2HealthyODLs, crossSiteAdminHealthy, crossSiteDbHealthy);
             }
             else {
                 // Make calls over to site 1 healthchecks
@@ -563,12 +578,13 @@ public class GrToolkitProvider implements AutoCloseable, GrToolkitService, DataT
                             if(!performedCrossSiteHealthCheck) {
                                 try {
                                     String content = getRequestContent(httpProtocol + clusterActor.getNode() + ":" + port + "/restconf/operations/gr-toolkit:site-identifier", HttpMethod.POST);
-                                    crossSiteIdentifier = new JSONObject(content).getJSONObject("output").getString("id");
+                                    crossSiteIdentifier = new JSONObject(content).getJSONObject(OUTPUT).getString("id");
                                     crossSiteDbHealthy = crossSiteHealthRequest(httpProtocol + clusterActor.getNode() + ":" + port + "/restconf/operations/gr-toolkit:database-health");
                                     crossSiteAdminHealthy = crossSiteHealthRequest(httpProtocol + clusterActor.getNode() + ":" + port + "/restconf/operations/gr-toolkit:admin-health");
                                     performedCrossSiteHealthCheck = true;
                                 } catch(Exception e) {
-                                    log.info("Cannot get site identifier from {}", clusterActor.getNode());
+                                    log.info("Cannot get cross site health from {}", clusterActor.getNode());
+                                    log.info("siteIdentifier: {} | dbHealth: {} | adminHealth: {}", crossSiteIdentifier, crossSiteDbHealthy, crossSiteAdminHealthy);
                                     log.error("Site Health Error", e);
                                 }
                             }
@@ -586,9 +602,12 @@ public class GrToolkitProvider implements AutoCloseable, GrToolkitService, DataT
                 outputBuilder.getSites().add(builder.build());
                 builder = getSitesBuilder(site2HealthyODLs, site2Voting, HEALTHY.equals(adminHealth), HEALTHY.equals(databaseHealth), siteIdentifier);
                 outputBuilder.getSites().add(builder.build());
+                log.info("{}:site-health: Site 1 ({}) | hasVotingMembers?: {} | Healthy ODLs: {} | ADM isHealthy?: {} | DB isHealthy?: {}", APP_NAME, siteIdentifier, site1Voting, site1HealthyODLs, HEALTHY.equals(adminHealth), HEALTHY.equals(databaseHealth));
+                log.info("{}:site-health: Site 2 ({}) | hasVotingMembers?: {} | Healthy ODLs: {} | ADM isHealthy?: {} | DB isHealthy?: {}", APP_NAME, crossSiteIdentifier, site2Voting, site2HealthyODLs, crossSiteAdminHealthy, crossSiteDbHealthy);
             }
         }
 
+        outputBuilder.setServedBy(member);
         RpcResult<SiteHealthOutput> rpcResult = RpcResultBuilder.<SiteHealthOutput>status(true).withResult(outputBuilder.build()).build();
         return Futures.immediateFuture(rpcResult);
     }
@@ -769,7 +788,7 @@ public class GrToolkitProvider implements AutoCloseable, GrToolkitService, DataT
                         (org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.halt.akka.traffic.input.NodeInfo) node;
                 log.info("Isolating {}", n.getNode());
                 executeCommand(String.format("sudo /sbin/iptables -A INPUT -p tcp --destination-port %s -j DROP -s %s", properties.get(PropertyKeys.CONTROLLER_PORT_AKKA), n.getNode()));
-                executeCommand(String.format("sudo /sbin/iptables -A OUTPUT -p tcp --destination-port %s -j DROP -s %s", n.getPort(), n.getNode()));
+                executeCommand(String.format("sudo /sbin/iptables -A OUTPUT -p tcp --destination-port %s -j DROP -d %s", n.getPort(), n.getNode()));
             }
 
         } else if(task == IpTables.DELETE) {
@@ -778,7 +797,7 @@ public class GrToolkitProvider implements AutoCloseable, GrToolkitService, DataT
                         (org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.resume.akka.traffic.input.NodeInfo) node;
                 log.info("De-isolating {}", n.getNode());
                 executeCommand(String.format("sudo /sbin/iptables -D INPUT -p tcp --destination-port %s -j DROP -s %s", properties.get(PropertyKeys.CONTROLLER_PORT_AKKA), n.getNode()));
-                executeCommand(String.format("sudo /sbin/iptables -D OUTPUT -p tcp --destination-port %s -j DROP -s %s", n.getPort(), n.getNode()));
+                executeCommand(String.format("sudo /sbin/iptables -D OUTPUT -p tcp --destination-port %s -j DROP -d %s", n.getPort(), n.getNode()));
             }
 
         }
@@ -807,7 +826,7 @@ public class GrToolkitProvider implements AutoCloseable, GrToolkitService, DataT
         String content = getRequestContent(path, HttpMethod.POST);
         try {
             JSONObject responseJson = new JSONObject(content);
-            JSONObject responseValue = responseJson.getJSONObject(VALUE);
+            JSONObject responseValue = responseJson.getJSONObject(OUTPUT);
             return HEALTHY.equals(responseValue.getString("health"));
         } catch(JSONException e) {
             log.error("Error parsing JSON", e);
@@ -836,9 +855,9 @@ public class GrToolkitProvider implements AutoCloseable, GrToolkitService, DataT
         log.info("Determining database health...");
         try {
             Connection connection = dbLib.getConnection();
-            log.info("DBLib isActive(): {}", dbLib.isActive());
-            log.info("DBLib isReadOnly(): {}", connection.isReadOnly());
-            log.info("DBLib isClosed(): {}", connection.isClosed());
+            log.debug("DBLib isActive(): {}", dbLib.isActive());
+            log.debug("DBLib isReadOnly(): {}", connection.isReadOnly());
+            log.debug("DBLib isClosed(): {}", connection.isClosed());
             if(!dbLib.isActive() || connection.isClosed() || connection.isReadOnly()) {
                 log.warn("Database is FAULTY");
                 connection.close();
@@ -876,7 +895,10 @@ public class GrToolkitProvider implements AutoCloseable, GrToolkitService, DataT
         }
         bufferedReader.close();
         connection.disconnect();
-        return content.toString();
+
+        String response = content.toString();
+        log.debug("getRequestContent(): Response:\n{}", response);
+        return response;
     }
 
     private int getRequestStatus(String path, HttpMethod method) throws IOException {
@@ -918,6 +940,8 @@ public class GrToolkitProvider implements AutoCloseable, GrToolkitService, DataT
         connection.addRequestProperty("Authorization", auth);
         connection.setRequestProperty("Connection", "keep-alive");
         connection.setRequestProperty("Proxy-Connection", "keep-alive");
+        connection.setConnectTimeout(CONNECTION_TIMEOUT);
+        connection.setReadTimeout(CONNECTION_TIMEOUT);
         return connection;
     }