/*- * ================================================================================ * eCOMP Portal * ================================================================================ * Copyright (C) 2017 AT&T Intellectual Property * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * ================================================================================ */ package org.openecomp.portalapp.portal.listener; import java.util.List; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import org.hibernate.Query; import org.hibernate.Session; import org.hibernate.SessionFactory; import org.openecomp.portalapp.portal.logging.aop.EPMetricsLog; import org.openecomp.portalapp.portal.logging.format.EPAppMessagesEnum; import org.openecomp.portalapp.portal.logging.logic.EPLogUtil; import org.openecomp.portalapp.portal.ueb.EPUebHelper; import org.openecomp.portalapp.portal.utils.EPSystemProperties; import org.openecomp.portalapp.portal.utils.EcompPortalUtils; import org.openecomp.portalsdk.core.logging.logic.EELFLoggerDelegate; import org.openecomp.portalsdk.core.util.SystemProperties; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.EnableAspectJAutoProxy; import org.springframework.transaction.annotation.Transactional; @Transactional @org.springframework.context.annotation.Configuration @EnableAspectJAutoProxy @EPMetricsLog public class HealthMonitor { @Autowired private SessionFactory sessionFactory; @Autowired private EPUebHelper epUebHelper; private static boolean databaseUp; private static boolean uebUp; private static boolean frontEndUp; private static boolean backEndUp; private static boolean dbClusterStatusOk; private static boolean dbPermissionsOk; public static boolean isSuspended = false; Thread healthMonitorThread; EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(HealthMonitor.class); public HealthMonitor() { } public static boolean isDatabaseUp() { return databaseUp; } public static boolean isClusterStatusOk() { return dbClusterStatusOk; } public static boolean isDatabasePermissionsOk() { return dbPermissionsOk; } public static boolean isUebUp() { return uebUp; } public static boolean isFrontEndUp() { return frontEndUp; } public static boolean isBackEndUp() { return backEndUp; } private void monitorEPHealth() throws InterruptedException{ int numIntervalsDatabaseHasBeenDown = 0; int numIntervalsClusterNotHealthy = 0; int numIntervalsDatabasePermissionsIncorrect = 0; int numIntervalsUebHasBeenDown = 0; logger.debug(EELFLoggerDelegate.debugLogger, "==> Health Monitor thread started"); long sleepInterval = (Long.valueOf(SystemProperties.getProperty(EPSystemProperties.HEALTH_POLL_INTERVAL_SECONDS)) * 1000); long numIntervalsBetweenAlerts = Long.valueOf(SystemProperties.getProperty(EPSystemProperties.HEALTHFAIL_ALERT_EVERY_X_INTERVALS)); logger.debug(EELFLoggerDelegate.debugLogger, "Polling health every " + sleepInterval + " milliseconds. Alerting every " + (sleepInterval * numIntervalsBetweenAlerts)/1000 + " seconds when component remains down."); while (true) { // // Get DB status. If down, signal alert once every X intervals. // databaseUp = this.checkIfDatabaseUp(); if (databaseUp == false) { if ((numIntervalsDatabaseHasBeenDown % numIntervalsBetweenAlerts) == 0){ // Write a Log entry that will generate an alert EPLogUtil.logEcompError(EPAppMessagesEnum.BeHealthCheckMySqlError); logger.debug(EELFLoggerDelegate.debugLogger, "Database down, logging to error log to trigger alert."); numIntervalsDatabaseHasBeenDown++; } else { numIntervalsDatabaseHasBeenDown = 0; } } dbClusterStatusOk = this.checkClusterStatus(); if (dbClusterStatusOk == false) { if ((numIntervalsClusterNotHealthy % numIntervalsBetweenAlerts) == 0){ EPLogUtil.logEcompError(EPAppMessagesEnum.BeHealthCheckMySqlError); logger.debug(EELFLoggerDelegate.debugLogger, "Cluster nodes appear to be down, logging to error log to trigger alert."); numIntervalsClusterNotHealthy++; } else { numIntervalsClusterNotHealthy = 0; } } dbPermissionsOk = this.checkDatabaseAndPermissions(); if (dbPermissionsOk == false) { if ((numIntervalsDatabasePermissionsIncorrect % numIntervalsBetweenAlerts) == 0){ EPLogUtil.logEcompError(EPAppMessagesEnum.BeHealthCheckMySqlError); logger.debug(EELFLoggerDelegate.debugLogger, "Database permissions don't seem correct, logging to error log to trigger alert."); numIntervalsDatabasePermissionsIncorrect++; } else { numIntervalsDatabasePermissionsIncorrect = 0; } } // // Get UEB status. Publish a bogus message to EP inbox, if 200 OK returned, status is Up. // If down, signal alert once every X intervals. // EP will ignore this bogus message. // uebUp = this.checkIfUebUp(); if (uebUp == false) { if ((numIntervalsUebHasBeenDown % numIntervalsBetweenAlerts) == 0){ // Write a Log entry that will generate an alert EPLogUtil.logEcompError(EPAppMessagesEnum.BeHealthCheckUebClusterError); logger.debug(EELFLoggerDelegate.debugLogger, "UEB down, logging to error log to trigger alert"); numIntervalsUebHasBeenDown++; } else { numIntervalsUebHasBeenDown = 0; } } //The front end should be up because the API is called through proxy front end server. frontEndUp = true; //If the rest API called, the backend is always up backEndUp = true; // // future nice to have...get Partner status // // For all apps exposing a rest url, query one of the rest urls(/roles?) and manage a list // of app name/status. We might not return back a non 200 OK in health check, but we // could return information in the json content of a health check. // // // Get DB status. If down, signal alert once every X intervals. // Thread.sleep(sleepInterval); } } @PostConstruct public void initHealthMonitor() { healthMonitorThread = new Thread("EP HealthMonitor thread") { public void run(){ try { monitorEPHealth(); } catch (InterruptedException e) { logger.debug(EELFLoggerDelegate.debugLogger, "<== Health Monitor thread exiting..." + EcompPortalUtils.getStackTrace(e)); } catch(Exception e) { logger.error(EELFLoggerDelegate.errorLogger, "Exception occurred in Healthcheck monitor thread." + EcompPortalUtils.getStackTrace(e)); } } }; if (healthMonitorThread != null) { healthMonitorThread.start(); } } @PreDestroy public void closeHealthMonitor() { this.healthMonitorThread.interrupt(); } private boolean checkIfDatabaseUp() { boolean isUp = false; Session localSession = null; try { localSession = sessionFactory.openSession(); if (localSession != null) { String sql = "select app_name from fn_app where app_id=1"; Query query = localSession.createSQLQuery(sql); @SuppressWarnings("unchecked") List queryList = query.list(); if (queryList != null) { isUp = true; } localSession.close(); } } catch (Exception e) { logger.error(EELFLoggerDelegate.errorLogger, EcompPortalUtils.getStackTrace(e)); isUp = false; } return isUp; } private boolean checkClusterStatus() { boolean isUp = false; Session localSession = null; try { localSession = sessionFactory.openSession(); if (localSession != null) { ///////////////////////////////////////////////////////////////////////// // If all nodes are unhealthy in a cluster, this will throw an exception ///////////////////////////////////////////////////////////////////////// String sql = "select * from mysql.user"; Query query = localSession.createSQLQuery(sql); @SuppressWarnings("unchecked") List queryList = query.list(); if (queryList != null) { isUp = true; } } } catch (Exception e) { logger.error(EELFLoggerDelegate.errorLogger, "checkClusterStatus() exception msg = " + e.getMessage()); if ((e.getCause() != null) && (e.getCause().getMessage() != null)) { logger.error(EELFLoggerDelegate.errorLogger, "checkClusterStatus() exception msg = " + e.getCause().getMessage()); } logger.error(EELFLoggerDelegate.errorLogger, "Exception inside checkClusterStatus() exception = " + EcompPortalUtils.getStackTrace(e)); isUp = false; } finally { if (localSession != null) { localSession.close(); } } return isUp; } private boolean checkDatabaseAndPermissions() { boolean isUp = false; Session localSession = null; try { localSession = sessionFactory.openSession(); if (localSession != null) { String sql = "SHOW GRANTS FOR CURRENT_USER"; Query query = localSession.createSQLQuery(sql); @SuppressWarnings("unchecked") List grantsList = query.list(); for (String str : grantsList) { if ((str.toUpperCase().contains("ALL")) || (str.toUpperCase().contains("DELETE") && str.toUpperCase().contains("SELECT") && str.toUpperCase().contains("UPDATE") && str.toUpperCase().contains("INSERT"))) { isUp = true; break; } } if (isUp == false) { logger.error(EELFLoggerDelegate.errorLogger, "checkDatabaseAndPermissions() returning false. SHOW GRANTS FOR CURRENT_USER being dumped:"); for (String str : grantsList) { logger.error(EELFLoggerDelegate.errorLogger, "grants output item = [" + str + "]"); } } } } catch (Exception e) { logger.error(EELFLoggerDelegate.errorLogger, "checkDatabaseAndPermissions() exception msg = " + e.getMessage()); if ((e.getCause() != null) && (e.getCause().getMessage() != null)) { logger.error(EELFLoggerDelegate.errorLogger, "checkDatabaseAndPermissions() exception msg = " + e.getCause().getMessage()); } logger.error(EELFLoggerDelegate.errorLogger, "Exception inside checkDatabaseAndPermissions() exception = " + EcompPortalUtils.getStackTrace(e)); isUp = false; } finally { if (localSession != null) { localSession.close(); } } return isUp; } private boolean checkIfUebUp() { boolean uebUp = false; try { boolean isAvailable = epUebHelper.checkAvailability(); boolean messageCanBeSent = epUebHelper.MessageCanBeSentToTopic(); uebUp = (isAvailable && messageCanBeSent); } catch (Exception e) { logger.error(EELFLoggerDelegate.errorLogger, "Exception inside CheckIfUebUp exception = " + EcompPortalUtils.getStackTrace(e)); } return uebUp; } /* private boolean checkIfFeUp() { boolean frontenndUp = false; try { String url = SystemProperties.getProperty(EPSystemProperties.FE_URL); if(StringUtils.isEmpty(url)) { logger.debug(EELFLoggerDelegate.debugLogger, "The front end URL is empty. Cannot check the status"); return frontenndUp; } url = url.replace("index.html", "app/healthCheck.json"); URL frontEndURL = new URL(url); HttpURLConnection con = (HttpURLConnection) frontEndURL.openConnection(); // optional default is GET con.setRequestMethod("GET"); int responseCode = con.getResponseCode(); logger.debug(EELFLoggerDelegate.debugLogger, "Fronend response code : " + responseCode); if(responseCode == 200) { frontenndUp = true; } } catch(Exception e){ logger.error(EELFLoggerDelegate.errorLogger, "Exception occurred while trying to access font end" + EcompPortalUtils.getStackTrace(e)); } return frontenndUp; } */ }