Add thread dump capabilities

A script provided can be executed at any time.
The programmatic solution will happen when
controllers (application) timeouts are detected.

Issue-ID: POLICY-4197
Signed-off-by: jhh <jorge.hernandez-herrero@att.com>
Change-Id: I1cca5f38e1f9b4f3fd7047e64dc58625ead5e1d1
Signed-off-by: jhh <jorge.hernandez-herrero@att.com>

diff --git a/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckManager.java b/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckManager.java
index b310168..59d0bfb 100644 (file)
--- a/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckManager.java
+++ b/feature-healthcheck/src/main/java/org/onap/policy/drools/healthcheck/HealthCheckManager.java
@@ -21,6 +21,7 @@
 package org.onap.policy.drools.healthcheck;
 
 import com.google.common.base.Strings;
+import java.lang.management.ManagementFactory;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -168,7 +169,7 @@ public class HealthCheckManager implements HealthCheck {
         }
 
         CompletableFuture<Report>[] reportFutures = futures(getControllers());
-        return summary(engineHealthcheck(), reportFutures);
+        return log(summary(engineHealthcheck(), reportFutures));
     }
 
     @Override
@@ -178,7 +179,7 @@ public class HealthCheckManager implements HealthCheck {
          * it could be useful for troubleshooting.
          */
         CompletableFuture<Report>[] reportFutures = futures(List.of(controller));
-        return summary(engineHealthcheck(), reportFutures);
+        return log(summary(engineHealthcheck(), reportFutures));
     }
 
     @Override
@@ -470,6 +471,23 @@ public class HealthCheckManager implements HealthCheck {
         return false;
     }
 
+    protected boolean isTimeout(Reports reports) {
+        return
+            reports.getDetails()
+                .stream()
+                .anyMatch(report -> report.getCode() == TIMEOUT_CODE);
+    }
+
+    protected Reports log(Reports reports) {
+        if (isTimeout(reports)) {
+            logger.warn("Healthcheck Timeout encountered");
+            Arrays.stream(ManagementFactory.getThreadMXBean()
+                    .dumpAllThreads(true, true, 100))
+                .forEach(threadInfo -> logger.info("Healthcheck Timeout Encountered:\n{}", threadInfo));
+        }
+        return reports;
+    }
+
     // the following methods may be overridden by junit tests
 
     protected PolicyEngine getEngineManager() {

diff --git a/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckManagerTest.java b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckManagerTest.java
index 73d70da..20cfa9e 100644 (file)
--- a/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckManagerTest.java
+++ b/feature-healthcheck/src/test/java/org/onap/policy/drools/healthcheck/HealthCheckManagerTest.java
@@ -238,6 +238,9 @@ public class HealthCheckManagerTest {
 
         /* with argument */
 
+        reports = monitor.controllerHealthcheck(controller1);
+        logger.info("{}", reports);
+
         reports = monitor.controllerHealthcheck(controller2);
         logger.info("{}", reports);
 

diff --git a/policy-management/src/main/server-gen/bin/jvm-dump b/policy-management/src/main/server-gen/bin/jvm-dump
new file mode 100644 (file)
index 0000000..0fa52e3
--- /dev/null
+++ b/policy-management/src/main/server-gen/bin/jvm-dump
@@ -0,0 +1,27 @@
+#!/usr/bin/env sh
+
+# ########################################################################
+# Copyright 2022 AT&T Intellectual Property. All rights reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ########################################################################
+
+
+if [ "${DEBUG}" = "y" ]; then
+    echo "-- $0 $* --"
+    set -x
+fi
+
+source "${POLICY_HOME}"/etc/profile.d/env.sh
+jstack -l -e $(pgrep java)
+