--- /dev/null
+/*-\r
+ * ================================================================================\r
+ * ECOMP Portal\r
+ * ================================================================================\r
+ * Copyright (C) 2017 AT&T Intellectual Property\r
+ * ================================================================================\r
+ * Licensed under the Apache License, Version 2.0 (the "License");\r
+ * you may not use this file except in compliance with the License.\r
+ * You may obtain a copy of the License at\r
+ * \r
+ * http://www.apache.org/licenses/LICENSE-2.0\r
+ * \r
+ * Unless required by applicable law or agreed to in writing, software\r
+ * distributed under the License is distributed on an "AS IS" BASIS,\r
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r
+ * See the License for the specific language governing permissions and\r
+ * limitations under the License.\r
+ * ================================================================================\r
+ */\r
+package org.openecomp.portalapp.portal.listener;\r
+\r
+import java.util.List;\r
+\r
+import javax.annotation.PostConstruct;\r
+import javax.annotation.PreDestroy;\r
+\r
+import org.hibernate.Query;\r
+import org.hibernate.Session;\r
+import org.hibernate.SessionFactory;\r
+import org.openecomp.portalapp.portal.logging.aop.EPMetricsLog;\r
+import org.openecomp.portalapp.portal.logging.format.EPAppMessagesEnum;\r
+import org.openecomp.portalapp.portal.logging.logic.EPLogUtil;\r
+import org.openecomp.portalapp.portal.ueb.EPUebHelper;\r
+import org.openecomp.portalapp.portal.utils.EPCommonSystemProperties;\r
+import org.openecomp.portalsdk.core.logging.logic.EELFLoggerDelegate;\r
+import org.openecomp.portalsdk.core.util.SystemProperties;\r
+import org.springframework.beans.factory.annotation.Autowired;\r
+import org.springframework.context.annotation.EnableAspectJAutoProxy;\r
+import org.springframework.transaction.annotation.Transactional;\r
+\r
+@Transactional\r
+@org.springframework.context.annotation.Configuration\r
+@EnableAspectJAutoProxy\r
+@EPMetricsLog\r
+public class HealthMonitor {\r
+\r
+ @Autowired\r
+ private SessionFactory sessionFactory;\r
+\r
+ @Autowired\r
+ private EPUebHelper epUebHelper;\r
+\r
+ private static boolean databaseUp;\r
+ private static boolean uebUp;\r
+ private static boolean frontEndUp;\r
+ private static boolean backEndUp;\r
+ private static boolean dbClusterStatusOk;\r
+ private static boolean dbPermissionsOk;\r
+ public static boolean isSuspended = false;\r
+\r
+ Thread healthMonitorThread;\r
+\r
+ EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(HealthMonitor.class);\r
+\r
+ public HealthMonitor() {\r
+\r
+ }\r
+\r
+ public static boolean isDatabaseUp() {\r
+ return databaseUp;\r
+ }\r
+\r
+ public static boolean isClusterStatusOk() {\r
+ return dbClusterStatusOk;\r
+ }\r
+\r
+ public static boolean isDatabasePermissionsOk() {\r
+ return dbPermissionsOk;\r
+ }\r
+\r
+ public static boolean isUebUp() {\r
+ return uebUp;\r
+ }\r
+\r
+ public static boolean isFrontEndUp() {\r
+ return frontEndUp;\r
+ }\r
+\r
+ public static boolean isBackEndUp() {\r
+ return backEndUp;\r
+ }\r
+\r
+ private void monitorEPHealth() throws InterruptedException {\r
+\r
+ int numIntervalsDatabaseHasBeenDown = 0;\r
+ int numIntervalsClusterNotHealthy = 0;\r
+ int numIntervalsDatabasePermissionsIncorrect = 0;\r
+ int numIntervalsUebHasBeenDown = 0;\r
+\r
+ logger.debug(EELFLoggerDelegate.debugLogger, "==> Health Monitor thread started");\r
+\r
+ long sleepInterval = (Long\r
+ .valueOf(SystemProperties.getProperty(EPCommonSystemProperties.HEALTH_POLL_INTERVAL_SECONDS)) * 1000);\r
+ long numIntervalsBetweenAlerts = Long\r
+ .valueOf(SystemProperties.getProperty(EPCommonSystemProperties.HEALTHFAIL_ALERT_EVERY_X_INTERVALS));\r
+ logger.debug(EELFLoggerDelegate.debugLogger,\r
+ "Polling health every " + sleepInterval + " milliseconds. Alerting every "\r
+ + (sleepInterval * numIntervalsBetweenAlerts) / 1000 + " seconds when component remains down.");\r
+\r
+ while (true) {\r
+ //\r
+ // Get DB status. If down, signal alert once every X intervals.\r
+ //\r
+ databaseUp = this.checkIfDatabaseUp();\r
+ if (databaseUp == false) {\r
+\r
+ if ((numIntervalsDatabaseHasBeenDown % numIntervalsBetweenAlerts) == 0) {\r
+ // Write a Log entry that will generate an alert\r
+ EPLogUtil.logEcompError(logger, EPAppMessagesEnum.BeHealthCheckMySqlError);\r
+ logger.debug(EELFLoggerDelegate.debugLogger,\r
+ "Database down, logging to error log to trigger alert.");\r
+ numIntervalsDatabaseHasBeenDown++;\r
+ } else {\r
+ numIntervalsDatabaseHasBeenDown = 0;\r
+ }\r
+ }\r
+\r
+ dbClusterStatusOk = this.checkClusterStatus();\r
+ if (dbClusterStatusOk == false) {\r
+\r
+ if ((numIntervalsClusterNotHealthy % numIntervalsBetweenAlerts) == 0) {\r
+ EPLogUtil.logEcompError(logger, EPAppMessagesEnum.BeHealthCheckMySqlError);\r
+ logger.debug(EELFLoggerDelegate.debugLogger,\r
+ "Cluster nodes appear to be down, logging to error log to trigger alert.");\r
+ numIntervalsClusterNotHealthy++;\r
+ } else {\r
+ numIntervalsClusterNotHealthy = 0;\r
+ }\r
+ }\r
+\r
+ dbPermissionsOk = this.checkDatabaseAndPermissions();\r
+ if (dbPermissionsOk == false) {\r
+\r
+ if ((numIntervalsDatabasePermissionsIncorrect % numIntervalsBetweenAlerts) == 0) {\r
+ EPLogUtil.logEcompError(logger, EPAppMessagesEnum.BeHealthCheckMySqlError);\r
+ logger.debug(EELFLoggerDelegate.debugLogger,\r
+ "Database permissions don't seem correct, logging to error log to trigger alert.");\r
+ numIntervalsDatabasePermissionsIncorrect++;\r
+ } else {\r
+ numIntervalsDatabasePermissionsIncorrect = 0;\r
+ }\r
+ }\r
+\r
+ //\r
+ // Get UEB status. Publish a bogus message to EP inbox, if 200 OK\r
+ // returned, status is Up.\r
+ // If down, signal alert once every X intervals.\r
+ // EP will ignore this bogus message.\r
+ //\r
+ uebUp = this.checkIfUebUp();\r
+ if (uebUp == false) {\r
+\r
+ if ((numIntervalsUebHasBeenDown % numIntervalsBetweenAlerts) == 0) {\r
+ // Write a Log entry that will generate an alert\r
+ EPLogUtil.logEcompError(logger, EPAppMessagesEnum.BeHealthCheckUebClusterError);\r
+ logger.debug(EELFLoggerDelegate.debugLogger, "UEB down, logging to error log to trigger alert");\r
+ numIntervalsUebHasBeenDown++;\r
+ } else {\r
+ numIntervalsUebHasBeenDown = 0;\r
+ }\r
+ }\r
+\r
+ // The front end should be up because the API is called through\r
+ // proxy front end server.\r
+ frontEndUp = true;\r
+\r
+ // If the rest API called, the backend is always up\r
+ backEndUp = true;\r
+\r
+ //\r
+ // future nice to have...get Partner status\r
+ //\r
+ // For all apps exposing a rest url, query one of the rest\r
+ // urls(/roles?) and manage a list\r
+ // of app name/status. We might not return back a non 200 OK in\r
+ // health check, but we\r
+ // could return information in the json content of a health check.\r
+ //\r
+\r
+ //\r
+ // Get DB status. If down, signal alert once every X intervals.\r
+ //\r
+ if (Thread.interrupted()) {\r
+ logger.info(EELFLoggerDelegate.errorLogger, "==> UebMainHandler exiting");\r
+ break;\r
+ }\r
+\r
+ try {\r
+ Thread.sleep(sleepInterval);\r
+ } catch (InterruptedException e) {\r
+ logger.error(EELFLoggerDelegate.errorLogger, "monitorEPHealth interrupted", e);\r
+ Thread.currentThread().interrupt();\r
+ }\r
+ }\r
+ }\r
+\r
+ @PostConstruct\r
+ public void initHealthMonitor() {\r
+\r
+ healthMonitorThread = new Thread("EP HealthMonitor thread") {\r
+ public void run() {\r
+ try {\r
+ monitorEPHealth();\r
+ } catch (InterruptedException e) {\r
+ logger.debug(EELFLoggerDelegate.debugLogger, "healthMonitorThread interrupted", e);\r
+ } catch (Exception e) {\r
+ logger.error(EELFLoggerDelegate.errorLogger, "healthMonitorThread failed", e);\r
+ }\r
+ }\r
+ };\r
+ if (healthMonitorThread != null) {\r
+ healthMonitorThread.start();\r
+ }\r
+ }\r
+\r
+ @PreDestroy\r
+ public void closeHealthMonitor() {\r
+ this.healthMonitorThread.interrupt();\r
+ }\r
+\r
+ private boolean checkIfDatabaseUp() {\r
+\r
+ boolean isUp = false;\r
+\r
+ Session localSession = null;\r
+\r
+ try {\r
+ localSession = sessionFactory.openSession();\r
+\r
+ if (localSession != null) {\r
+\r
+ String sql = "select app_name from fn_app where app_id=1";\r
+ Query query = localSession.createSQLQuery(sql);\r
+ @SuppressWarnings("unchecked")\r
+ List<String> queryList = query.list();\r
+ if (queryList != null) {\r
+ isUp = true;\r
+ }\r
+ localSession.close();\r
+ }\r
+ } catch (Exception e) {\r
+ logger.debug(EELFLoggerDelegate.debugLogger, "checkIfDatabaseUp failed", e);\r
+ isUp = false;\r
+ }\r
+\r
+ return isUp;\r
+ }\r
+\r
+ private boolean checkClusterStatus() {\r
+\r
+ boolean isUp = false;\r
+\r
+ Session localSession = null;\r
+\r
+ try {\r
+ localSession = sessionFactory.openSession();\r
+ if (localSession != null) {\r
+ // If all nodes are unhealthy in a cluster, this will throw an\r
+ // exception\r
+ String sql = "select * from mysql.user";\r
+ Query query = localSession.createSQLQuery(sql);\r
+ @SuppressWarnings("unchecked")\r
+ List<String> queryList = query.list();\r
+ if (queryList != null) {\r
+ isUp = true;\r
+ }\r
+ }\r
+ } catch (Exception e) {\r
+ logger.error(EELFLoggerDelegate.errorLogger, "checkClusterStatus failed", e);\r
+ if ((e.getCause() != null) && (e.getCause().getMessage() != null)) {\r
+ logger.error(EELFLoggerDelegate.errorLogger,\r
+ "checkClusterStatus() exception msg = " + e.getCause().getMessage());\r
+ }\r
+ isUp = false;\r
+ } finally {\r
+ if (localSession != null) {\r
+ localSession.close();\r
+ }\r
+ }\r
+\r
+ return isUp;\r
+\r
+ }\r
+\r
+ private boolean checkDatabaseAndPermissions() {\r
+\r
+ boolean isUp = false;\r
+\r
+ Session localSession = null;\r
+\r
+ try {\r
+ localSession = sessionFactory.openSession();\r
+ if (localSession != null) {\r
+ String sql = "SHOW GRANTS FOR CURRENT_USER";\r
+ Query query = localSession.createSQLQuery(sql);\r
+ @SuppressWarnings("unchecked")\r
+ List<String> grantsList = query.list();\r
+ for (String str : grantsList) {\r
+ if ((str.toUpperCase().contains("ALL"))\r
+ || (str.toUpperCase().contains("DELETE") && str.toUpperCase().contains("SELECT")\r
+ && str.toUpperCase().contains("UPDATE") && str.toUpperCase().contains("INSERT"))) {\r
+ isUp = true;\r
+ break;\r
+ }\r
+ }\r
+ if (isUp == false) {\r
+ logger.error(EELFLoggerDelegate.errorLogger,\r
+ "checkDatabaseAndPermissions() returning false. SHOW GRANTS FOR CURRENT_USER being dumped:");\r
+ for (String str : grantsList) {\r
+ logger.error(EELFLoggerDelegate.errorLogger, "grants output item = [" + str + "]");\r
+ }\r
+ }\r
+ }\r
+ } catch (Exception e) {\r
+ logger.error(EELFLoggerDelegate.errorLogger, "checkDatabaseAndPermissions failed", e);\r
+ if ((e.getCause() != null) && (e.getCause().getMessage() != null)) {\r
+ logger.error(EELFLoggerDelegate.errorLogger,\r
+ "checkDatabaseAndPermissions() exception msg = " + e.getCause().getMessage());\r
+ }\r
+ isUp = false;\r
+ } finally {\r
+ if (localSession != null) {\r
+ localSession.close();\r
+ }\r
+ }\r
+\r
+ return isUp;\r
+\r
+ }\r
+\r
+ private boolean checkIfUebUp() {\r
+ boolean uebUp = false;\r
+ try {\r
+ boolean isAvailable = epUebHelper.checkAvailability();\r
+ boolean messageCanBeSent = epUebHelper.MessageCanBeSentToTopic();\r
+ uebUp = (isAvailable && messageCanBeSent);\r
+ } catch (Exception e) {\r
+ logger.error(EELFLoggerDelegate.errorLogger, "checkIfUebUp failed", e);\r
+ }\r
+\r
+ return uebUp;\r
+\r
+ }\r
+\r
+}\r
Code Review / portal.git / blobdiff