X-Git-Url: https://gerrit.onap.org/r/gitweb?a=blobdiff_plain;f=ecomp-portal-BE-common%2Fsrc%2Fmain%2Fjava%2Forg%2Fonap%2Fportalapp%2Fportal%2Flistener%2FHealthMonitor.java;h=2d18d751386b56c25604d73031e321b046904cff;hb=HEAD;hp=6f934e984a9f878bb018777daab8fa291290e1da;hpb=d84a85d705b38d90b73809ead3e5034b8c066ca9;p=portal.git diff --git a/ecomp-portal-BE-common/src/main/java/org/onap/portalapp/portal/listener/HealthMonitor.java b/ecomp-portal-BE-common/src/main/java/org/onap/portalapp/portal/listener/HealthMonitor.java index 6f934e98..2d18d751 100644 --- a/ecomp-portal-BE-common/src/main/java/org/onap/portalapp/portal/listener/HealthMonitor.java +++ b/ecomp-portal-BE-common/src/main/java/org/onap/portalapp/portal/listener/HealthMonitor.java @@ -2,7 +2,7 @@ * ============LICENSE_START========================================== * ONAP Portal * =================================================================== - * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * Copyright (C) 2017-2018 AT&T Intellectual Property. All rights reserved. * =================================================================== * * Unless otherwise specified, all software contained herein is licensed @@ -37,17 +37,14 @@ */ package org.onap.portalapp.portal.listener; -import java.io.IOException; -import java.util.HashSet; +import java.time.Instant; import java.util.List; -import java.util.Set; -import java.util.UUID; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; +import lombok.NoArgsConstructor; import org.apache.commons.lang3.StringUtils; -import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.client.FourLetterWordMain; import org.hibernate.Query; import org.hibernate.Session; @@ -64,6 +61,7 @@ import org.onap.portalapp.portal.utils.EPCommonSystemProperties; import org.onap.portalsdk.core.logging.logic.EELFLoggerDelegate; import org.onap.portalsdk.core.util.SystemProperties; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.EnableAspectJAutoProxy; import org.springframework.transaction.annotation.Transactional; @@ -71,95 +69,56 @@ import org.springframework.transaction.annotation.Transactional; @Transactional -@org.springframework.context.annotation.Configuration +@Configuration @EnableAspectJAutoProxy @EPMetricsLog +@NoArgsConstructor public class HealthMonitor { - - - ZooKeeper zookeeper = null; - private static EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(HealthMonitor.class); - - @Autowired - private SessionFactory sessionFactory; - + private Thread healthMonitorThread; + private static SessionFactory sessionFactory; private static boolean databaseUp; private static boolean uebUp; private static boolean frontEndUp; private static boolean backEndUp; - private static boolean dbClusterStatusOk; private static boolean dbPermissionsOk; private static boolean zookeeperStatusOk; private static boolean cassandraStatusOk; - - /** - * Read directly by external classes. - */ - public static boolean isSuspended = false; - - private Thread healthMonitorThread; - - public HealthMonitor() { - } - - public static boolean isDatabaseUp() { - return databaseUp; - } - - public static boolean isClusterStatusOk() { - return dbClusterStatusOk; - } + private static String application = "Portal"; + private static boolean isSuspended = false; - public static boolean isDatabasePermissionsOk() { - return dbPermissionsOk; - } - - public static boolean isUebUp() { - return uebUp; - } - - public static boolean isFrontEndUp() { - return frontEndUp; - } - - public static boolean isBackEndUp() { - return backEndUp; - } - - public static boolean isZookeeperStatusOk() { - return zookeeperStatusOk; - } - - public static boolean isCassandraStatusOk() { - return cassandraStatusOk; + @Autowired + public HealthMonitor(SessionFactory sessionFactory) { + HealthMonitor.sessionFactory = sessionFactory; } - private void monitorEPHealth() throws InterruptedException { + private static void monitorEPHealth() { int numIntervalsDatabaseHasBeenDown = 0; - int numIntervalsClusterNotHealthy = 0; int numIntervalsDatabasePermissionsIncorrect = 0; int numIntervalsZookeeperNotHealthy = 0; int numIntervalsCassandraNotHealthy = 0; logger.debug(EELFLoggerDelegate.debugLogger, "monitorEPHealth thread started"); + long sleepInterval = (Long - .valueOf(SystemProperties.getProperty(EPCommonSystemProperties.HEALTH_POLL_INTERVAL_SECONDS)) * 1000); + .parseLong(SystemProperties.getProperty(EPCommonSystemProperties.HEALTH_POLL_INTERVAL_SECONDS)) * 1000); long numIntervalsBetweenAlerts = Long - .valueOf(SystemProperties.getProperty(EPCommonSystemProperties.HEALTHFAIL_ALERT_EVERY_X_INTERVALS)); + .parseLong(SystemProperties.getProperty(EPCommonSystemProperties.HEALTHFAIL_ALERT_EVERY_X_INTERVALS)); logger.debug(EELFLoggerDelegate.debugLogger, "monitorEPHealth: Polling health every " + sleepInterval + " milliseconds. Alerting every " + (sleepInterval * numIntervalsBetweenAlerts) / 1000 + " seconds when component remains down."); - + while (true) { + logger.debug(EELFLoggerDelegate.debugLogger, + "monitorEPHealth: Test Connection to all"); // // Get DB status. If down, signal alert once every X intervals. // - databaseUp = this.checkIfDatabaseUp(); - if (databaseUp == false) { + databaseUp = checkIfDatabaseUp(); + if (databaseUp) { if ((numIntervalsDatabaseHasBeenDown % numIntervalsBetweenAlerts) == 0) { logger.debug(EELFLoggerDelegate.debugLogger, "monitorEPHealth: database down, logging to error log to trigger alert."); @@ -171,20 +130,8 @@ public class HealthMonitor { } } - dbClusterStatusOk = this.checkClusterStatus(); - if (dbClusterStatusOk == false) { - if ((numIntervalsClusterNotHealthy % numIntervalsBetweenAlerts) == 0) { - logger.debug(EELFLoggerDelegate.debugLogger, - "monitorEPHealth: cluster nodes down, logging to error log to trigger alert."); - EPLogUtil.logEcompError(logger, EPAppMessagesEnum.BeHealthCheckMySqlError); - numIntervalsClusterNotHealthy++; - } else { - numIntervalsClusterNotHealthy = 0; - } - } - - dbPermissionsOk = this.checkDatabasePermissions(); - if (dbPermissionsOk == false) { + dbPermissionsOk = checkDatabasePermissions(); + if (!dbPermissionsOk) { if ((numIntervalsDatabasePermissionsIncorrect % numIntervalsBetweenAlerts) == 0) { logger.debug(EELFLoggerDelegate.debugLogger, "monitorEPHealth: database permissions incorrect, logging to error log to trigger alert."); @@ -194,68 +141,36 @@ public class HealthMonitor { numIntervalsDatabasePermissionsIncorrect = 0; } } - - zookeeperStatusOk = this.checkZookeeperStatus(); - if (zookeeperStatusOk == false) { - if ((numIntervalsZookeeperNotHealthy % numIntervalsBetweenAlerts) == 0) { - logger.debug(EELFLoggerDelegate.debugLogger, - "monitorEPHealth: cluster nodes down, logging to error log to trigger alert."); - EPLogUtil.logEcompError(logger, EPAppMessagesEnum.MusicHealthCheckZookeeperError); - numIntervalsZookeeperNotHealthy++; - } else { - numIntervalsZookeeperNotHealthy = 0; - } - } - - cassandraStatusOk = this.checkCassandraStatus(); - if (cassandraStatusOk == false) { - if ((numIntervalsCassandraNotHealthy % numIntervalsBetweenAlerts) == 0) { - logger.debug(EELFLoggerDelegate.debugLogger, - "monitorEPHealth: cluster nodes down, logging to error log to trigger alert."); - EPLogUtil.logEcompError(logger, EPAppMessagesEnum.MusicHealthCheckCassandraError); - numIntervalsCassandraNotHealthy++; - } else { - numIntervalsCassandraNotHealthy = 0; + if(org.onap.portalapp.music.util.MusicUtil.isMusicEnable()){ + + /* + * zookeeperStatusOk = checkZookeeperStatus(); + * + * if (!zookeeperStatusOk) { if ((numIntervalsZookeeperNotHealthy % + * numIntervalsBetweenAlerts) == 0) { + * logger.debug(EELFLoggerDelegate.debugLogger, + * "monitorEPHealth: cluster nodes down, logging to error log to trigger alert." + * ); EPLogUtil.logEcompError(logger, + * EPAppMessagesEnum.MusicHealthCheckZookeeperError); + * numIntervalsZookeeperNotHealthy++; } else { numIntervalsZookeeperNotHealthy = + * 0; } } + */ + + cassandraStatusOk = checkCassandraStatus(); + if (!cassandraStatusOk) { + if ((numIntervalsCassandraNotHealthy % numIntervalsBetweenAlerts) == 0) { + logger.debug(EELFLoggerDelegate.debugLogger, + "monitorEPHealth: cluster nodes down, logging to error log to trigger alert."); + EPLogUtil.logEcompError(logger, EPAppMessagesEnum.MusicHealthCheckCassandraError); + numIntervalsCassandraNotHealthy++; + } else { + numIntervalsCassandraNotHealthy = 0; + } } } - - // - // Get UEB status. Publish a bogus message to EP inbox, if 200 OK - // returned, status is Up. - // If down, signal alert once every X intervals. - // EP will ignore this bogus message. - // Commenting this out as Dependency on UEB is being deprecated - /* - * uebUp = this.checkIfUebUp(); if (uebUp == false) { - * - * if ((numIntervalsUebHasBeenDown % numIntervalsBetweenAlerts) == 0) { - * logger.debug(EELFLoggerDelegate.debugLogger, - * "monitorEPHealth: UEB down, logging to error log to trigger alert"); // Write - * a Log entry that will generate an alert EPLogUtil.logEcompError(logger, - * EPAppMessagesEnum.BeHealthCheckUebClusterError); - * numIntervalsUebHasBeenDown++; } else { numIntervalsUebHasBeenDown = 0; } } - */ - - // The front end should be up because the API is called through - // proxy front end server. frontEndUp = true; - - // If the rest API called, the backend is always up backEndUp = true; - // - // future nice to have...get Partner status - // - // For all apps exposing a rest url, query one of the rest - // urls(/roles?) and manage a list - // of app name/status. We might not return back a non 200 OK in - // health check, but we - // could return information in the json content of a health check. - // - - // - // Get DB status. If down, signal alert once every X intervals. - // if (Thread.interrupted()) { logger.info(EELFLoggerDelegate.errorLogger, "monitorEPHealth: thread interrupted"); break; @@ -273,12 +188,12 @@ public class HealthMonitor { @PostConstruct public void initHealthMonitor() { healthMonitorThread = new Thread("EP HealthMonitor thread") { + @Override public void run() { try { monitorEPHealth(); - } catch (InterruptedException e) { - logger.debug(EELFLoggerDelegate.debugLogger, "healthMonitorThread interrupted", e); - } catch (Exception e) { + } + catch (Exception e) { logger.error(EELFLoggerDelegate.errorLogger, "healthMonitorThread failed", e); } } @@ -302,7 +217,7 @@ public class HealthMonitor { * * @return true if the database can be read. */ - private boolean checkIfDatabaseUp() { + private static boolean checkIfDatabaseUp() { boolean isUp = false; Session localSession = null; try { @@ -326,55 +241,27 @@ public class HealthMonitor { return isUp; } - private boolean checkClusterStatus() { - boolean isUp = false; - Session localSession = null; - try { - localSession = sessionFactory.openSession(); - if (localSession != null) { - // If all nodes are unhealthy in a cluster, this will throw an - // exception - String sql = "select * from mysql.user"; - Query query = localSession.createSQLQuery(sql); - @SuppressWarnings("unchecked") - List queryList = query.list(); - if (queryList != null) { - isUp = true; - } - } - } catch (Exception e) { - logger.error(EELFLoggerDelegate.errorLogger, "checkClusterStatus failed", e); - if ((e.getCause() != null) && (e.getCause().getMessage() != null)) { - logger.error(EELFLoggerDelegate.errorLogger, "checkClusterStatus failure cause", e.getCause()); - } - isUp = false; - } finally { - if (localSession != null) { - localSession.close(); - } - } - return isUp; - } - - private boolean checkZookeeperStatus() { + private static boolean checkZookeeperStatus() { String[] zookeeperNodes = MusicUtil.getMyZkHost().split(","); logger.info(EELFLoggerDelegate.applicationLogger, "MusicUtil.getMyZkHost()---- :" + MusicUtil.getMyZkHost()); - for (int i = 0; i < zookeeperNodes.length; i++) { + for (String zookeeperNode : zookeeperNodes) { try { - logger.info(EELFLoggerDelegate.applicationLogger, "server ip--zookeeper :" + zookeeperNodes[i].trim()); - String[] iport = zookeeperNodes[i].split(":"); + logger.info(EELFLoggerDelegate.applicationLogger, "server ip--zookeeper :" + zookeeperNode.trim()); + String[] iport = zookeeperNode.split(":"); String zkNodeStatistics = FourLetterWordMain.send4LetterWord(iport[0].trim(), - Integer.parseInt(iport[1].trim()), "stat"); + Integer.parseInt(iport[1].trim()), "stat"); logger.info(EELFLoggerDelegate.applicationLogger, - "Getting Status for Zookeeper zkNodeStatistics :" + zkNodeStatistics); + "Getting Status for Zookeeper zkNodeStatistics :" + zkNodeStatistics); if (StringUtils.isNotBlank(zkNodeStatistics)) { String state = zkNodeStatistics.substring(zkNodeStatistics.indexOf("Mode:"), - zkNodeStatistics.indexOf("Node")); + zkNodeStatistics.indexOf("Node")); logger.info(EELFLoggerDelegate.applicationLogger, - "Getting Status for zookeeper :" + zookeeperNodes[i].trim() + ":------:" + state); - if (state.contains("leader")) + + "Getting Status for zookeeper :" + zookeeperNode.trim() + ":------:" + state); + if (state.contains("leader") || state.contains("follower")) { return true; + } } } catch (Exception e) { logger.error(EELFLoggerDelegate.errorLogger, "ZookeeperStatus Service is not responding", e.getCause()); @@ -385,9 +272,9 @@ public class HealthMonitor { } - public boolean checkCassandraStatus() { + private static boolean checkCassandraStatus() { logger.info(EELFLoggerDelegate.applicationLogger, "Getting Status for Cassandra"); - if (this.getAdminKeySpace()) { + if (getAdminKeySpace()) { return true; } else { logger.error(EELFLoggerDelegate.errorLogger, "Cassandra Service is not responding"); @@ -395,33 +282,29 @@ public class HealthMonitor { } } - private Boolean getAdminKeySpace() { - String musicKeySpace = MusicProperties.getProperty(MusicProperties.MUSIC_SESSION_KEYSPACE ); - //deletePortalHealthcheck(musicKeySpace); + private static Boolean getAdminKeySpace() { + String musicKeySpace = MusicProperties.getProperty(MusicProperties.MUSIC_SESSION_KEYSPACE); + Instant creationTime = Instant.now(); PreparedQueryObject pQuery = new PreparedQueryObject(); - pQuery.appendQueryString("insert into "+musicKeySpace+".healthcheck (id) values (?)"); - pQuery.addValue(UUID.randomUUID()); + pQuery.appendQueryString( + "UPDATE " + musicKeySpace + ".health_check SET creation_time = ? WHERE primary_id = ?"); + pQuery.addValue(creationTime.toString()); + pQuery.addValue(application); try { - MusicCore.nonKeyRelatedPut(pQuery, MusicUtil.EVENTUAL); + MusicCore.nonKeyRelatedPut(pQuery, MusicUtil.CRITICAL); } catch (MusicServiceException e) { - logger.error(EELFLoggerDelegate.errorLogger, "getAdminKeySpace() failed", e.getCause()); + logger.error(EELFLoggerDelegate.errorLogger, e.getErrorMessage(), e); + return Boolean.FALSE; + } catch (Exception ex) { + logger.error(EELFLoggerDelegate.errorLogger, ex.getMessage(), ex); return Boolean.FALSE; } - return Boolean.TRUE; - } + return Boolean.TRUE; - - private void deletePortalHealthcheck(String musicKeySpace) { - PreparedQueryObject pQuery = new PreparedQueryObject(); - pQuery.appendQueryString("TRUNCATE "+musicKeySpace+".healthcheck"); - try { - MusicCore.nonKeyRelatedPut(pQuery, MusicUtil.EVENTUAL); - } catch (MusicServiceException e) { - logger.error(EELFLoggerDelegate.errorLogger, "deletePortalHealthcheck() failed", e.getCause()); - } } + - private boolean checkDatabasePermissions() { + private static boolean checkDatabasePermissions() { boolean isUp = false; Session localSession = null; try { @@ -439,7 +322,7 @@ public class HealthMonitor { break; } } - if (isUp == false) { + if (!isUp) { logger.error(EELFLoggerDelegate.errorLogger, "checkDatabasePermissions returning false. SHOW GRANTS FOR CURRENT_USER being dumped:"); for (String str : grantsList) { @@ -460,5 +343,40 @@ public class HealthMonitor { } return isUp; } - + + public static boolean isDatabaseUp() { + return databaseUp; + } + + public static boolean isUebUp() { + return uebUp; + } + + public static boolean isFrontEndUp() { + return frontEndUp; + } + + public static boolean isBackEndUp() { + return backEndUp; + } + + public static boolean isDbPermissionsOk() { + return dbPermissionsOk; + } + + public static boolean isZookeeperStatusOk() { + return zookeeperStatusOk; + } + + public static boolean isCassandraStatusOk() { + return cassandraStatusOk; + } + + public static boolean isSuspended() { + return isSuspended; + } + + public static void setSuspended(boolean isSuspended) { + HealthMonitor.isSuspended = isSuspended; + } }