ecomp-portal-BE-common/src/main/java/org/openecomp/portalapp/portal/listener/HealthMonitor.java

   1 /*-
   2  * ================================================================================
   3  * ECOMP Portal
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ================================================================================
  19  */
  20 package org.openecomp.portalapp.portal.listener;
  21
  22 import java.util.Date;
  23 import java.util.List;
  24
  25 import javax.annotation.PostConstruct;
  26 import javax.annotation.PreDestroy;
  27
  28 import org.hibernate.Query;
  29 import org.hibernate.Session;
  30 import org.hibernate.SessionFactory;
  31 import org.openecomp.portalapp.portal.domain.SharedContext;
  32 import org.openecomp.portalapp.portal.logging.aop.EPMetricsLog;
  33 import org.openecomp.portalapp.portal.logging.format.EPAppMessagesEnum;
  34 import org.openecomp.portalapp.portal.logging.logic.EPLogUtil;
  35 import org.openecomp.portalapp.portal.service.SharedContextService;
  36 import org.openecomp.portalapp.portal.ueb.EPUebHelper;
  37 import org.openecomp.portalapp.portal.utils.EPCommonSystemProperties;
  38 import org.openecomp.portalsdk.core.logging.logic.EELFLoggerDelegate;
  39 import org.openecomp.portalsdk.core.util.SystemProperties;
  40 import org.springframework.beans.factory.annotation.Autowired;
  41 import org.springframework.context.annotation.EnableAspectJAutoProxy;
  42 import org.springframework.transaction.annotation.Transactional;
  43
  44 @Transactional
  45 @org.springframework.context.annotation.Configuration
  46 @EnableAspectJAutoProxy
  47 @EPMetricsLog
  48 public class HealthMonitor {
  49
  50         @Autowired
  51         private SessionFactory sessionFactory;
  52
  53         @Autowired
  54         private EPUebHelper epUebHelper;
  55
  56         @Autowired
  57         private SharedContextService sharedContextService;
  58
  59         private static boolean databaseUp;
  60         private static boolean uebUp;
  61         private static boolean frontEndUp;
  62         private static boolean backEndUp;
  63         private static boolean dbClusterStatusOk;
  64         private static boolean dbPermissionsOk;
  65         public static boolean isSuspended = false;
  66
  67         private Thread healthMonitorThread;
  68
  69         private EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(HealthMonitor.class);
  70
  71         public HealthMonitor() {
  72
  73         }
  74
  75         public static boolean isDatabaseUp() {
  76                 return databaseUp;
  77         }
  78
  79         public static boolean isClusterStatusOk() {
  80                 return dbClusterStatusOk;
  81         }
  82
  83         public static boolean isDatabasePermissionsOk() {
  84                 return dbPermissionsOk;
  85         }
  86
  87         public static boolean isUebUp() {
  88                 return uebUp;
  89         }
  90
  91         public static boolean isFrontEndUp() {
  92                 return frontEndUp;
  93         }
  94
  95         public static boolean isBackEndUp() {
  96                 return backEndUp;
  97         }
  98
  99         private void monitorEPHealth() throws InterruptedException {
 100
 101                 int numIntervalsDatabaseHasBeenDown = 0;
 102                 int numIntervalsClusterNotHealthy = 0;
 103                 int numIntervalsDatabasePermissionsIncorrect = 0;
 104                 int numIntervalsUebHasBeenDown = 0;
 105
 106                 logger.debug(EELFLoggerDelegate.debugLogger, "monitorEPHealth started");
 107
 108                 long sleepInterval = (Long
 109                                 .valueOf(SystemProperties.getProperty(EPCommonSystemProperties.HEALTH_POLL_INTERVAL_SECONDS)) * 1000);
 110                 long numIntervalsBetweenAlerts = Long
 111                                 .valueOf(SystemProperties.getProperty(EPCommonSystemProperties.HEALTHFAIL_ALERT_EVERY_X_INTERVALS));
 112                 logger.debug(EELFLoggerDelegate.debugLogger,
 113                                 "monitorEPHealth: Polling health every " + sleepInterval + " milliseconds. Alerting every "
 114                                                 + (sleepInterval * numIntervalsBetweenAlerts) / 1000 + " seconds when component remains down.");
 115
 116                 while (true) {
 117                         //
 118                         // Get DB status. If down, signal alert once every X intervals.
 119                         //
 120                         databaseUp = this.checkIfDatabaseUp();
 121                         if (databaseUp == false) {
 122                                 if ((numIntervalsDatabaseHasBeenDown % numIntervalsBetweenAlerts) == 0) {
 123                                         logger.debug(EELFLoggerDelegate.debugLogger,
 124                                                         "monitorEPHealth: database down, logging to error log to trigger alert.");
 125                                         // Write a Log entry that will generate an alert
 126                                         EPLogUtil.logEcompError(logger, EPAppMessagesEnum.BeHealthCheckMySqlError);
 127                                         numIntervalsDatabaseHasBeenDown++;
 128                                 } else {
 129                                         numIntervalsDatabaseHasBeenDown = 0;
 130                                 }
 131                         }
 132
 133                         dbClusterStatusOk = this.checkClusterStatus();
 134                         if (dbClusterStatusOk == false) {
 135                                 if ((numIntervalsClusterNotHealthy % numIntervalsBetweenAlerts) == 0) {
 136                                         logger.debug(EELFLoggerDelegate.debugLogger,
 137                                                         "monitorEPHealth: cluster nodes down, logging to error log to trigger alert.");
 138                                         EPLogUtil.logEcompError(logger, EPAppMessagesEnum.BeHealthCheckMySqlError);
 139                                         numIntervalsClusterNotHealthy++;
 140                                 } else {
 141                                         numIntervalsClusterNotHealthy = 0;
 142                                 }
 143                         }
 144
 145                         dbPermissionsOk = this.checkDatabasePermissions();
 146                         if (dbPermissionsOk == false) {
 147                                 if ((numIntervalsDatabasePermissionsIncorrect % numIntervalsBetweenAlerts) == 0) {
 148                                         logger.debug(EELFLoggerDelegate.debugLogger,
 149                                                         "monitorEPHealth: database permissions not correct, logging to error log to trigger alert.");
 150                                         EPLogUtil.logEcompError(logger, EPAppMessagesEnum.BeHealthCheckMySqlError);
 151                                         numIntervalsDatabasePermissionsIncorrect++;
 152                                 } else {
 153                                         numIntervalsDatabasePermissionsIncorrect = 0;
 154                                 }
 155                         }
 156
 157                         //
 158                         // Get UEB status. Publish a bogus message to EP inbox, if 200 OK
 159                         // returned, status is Up.
 160                         // If down, signal alert once every X intervals.
 161                         // EP will ignore this bogus message.
 162                         //
 163                         uebUp = this.checkIfUebUp();
 164                         if (uebUp == false) {
 165                                 if ((numIntervalsUebHasBeenDown % numIntervalsBetweenAlerts) == 0) {
 166                                         logger.debug(EELFLoggerDelegate.debugLogger, "UEB down, logging to error log to trigger alert");
 167                                         // Write a Log entry that will generate an alert
 168                                         EPLogUtil.logEcompError(logger, EPAppMessagesEnum.BeHealthCheckUebClusterError);
 169                                         numIntervalsUebHasBeenDown++;
 170                                 } else {
 171                                         numIntervalsUebHasBeenDown = 0;
 172                                 }
 173                         }
 174
 175                         // The front end should be up because the API is called through
 176                         // proxy front end server.
 177                         frontEndUp = true;
 178
 179                         // If the rest API called, the backend is always up
 180                         backEndUp = true;
 181
 182                         //
 183                         // future nice to have...get Partner status
 184                         //
 185                         // For all apps exposing a rest url, query one of the rest
 186                         // urls(/roles?) and manage a list
 187                         // of app name/status. We might not return back a non 200 OK in
 188                         // health check, but we
 189                         // could return information in the json content of a health check.
 190                         //
 191
 192                         if (Thread.interrupted()) {
 193                                 logger.debug(EELFLoggerDelegate.debugLogger, "monitorEPHealth: interrupted, leaving loop");
 194                                 break;
 195                         }
 196
 197                         try {
 198                                 Thread.sleep(sleepInterval);
 199                         } catch (InterruptedException e) {
 200                                 logger.error(EELFLoggerDelegate.errorLogger, "monitorEPHealth interrupted", e);
 201                                 Thread.currentThread().interrupt();
 202                         }
 203                 }
 204         }
 205
 206         @PostConstruct
 207         public void initHealthMonitor() {
 208
 209                 healthMonitorThread = new Thread("EP HealthMonitor thread") {
 210                         public void run() {
 211                                 try {
 212                                         monitorEPHealth();
 213                                 } catch (InterruptedException e) {
 214                                         logger.debug(EELFLoggerDelegate.debugLogger, "healthMonitorThread interrupted", e);
 215                                 } catch (Exception e) {
 216                                         logger.error(EELFLoggerDelegate.errorLogger, "healthMonitorThread failed", e);
 217                                 }
 218                         }
 219                 };
 220                 if (healthMonitorThread != null) {
 221                         healthMonitorThread.start();
 222                 }
 223         }
 224
 225         @PreDestroy
 226         public void closeHealthMonitor() {
 227                 this.healthMonitorThread.interrupt();
 228         }
 229
 230         /**
 231          * Writes and reads the database; cleans up when finished.
 232          *
 233          * @return True on success; false otherwise.
 234          */
 235         private boolean checkIfDatabaseUp() {
 236                 boolean isUp = false;
 237                 try {
 238                         final Date now = new Date();
 239                         final String contextId = "checkIfDatabaseUp-" + Long.toString(now.getTime());
 240                         final String key = "checkIfDatabaseUp-key";
 241                         final String value = "checkIfDatabaseUp-value";
 242                         sharedContextService.addSharedContext(contextId, key, value);
 243                         SharedContext sc = sharedContextService.getSharedContext(contextId, key);
 244                         if (sc == null || sc.getCvalue() == null || !value.equals(sc.getCvalue()))
 245                                 throw new Exception("Failed to retrieve shared context");
 246                         int removed = sharedContextService.deleteSharedContexts(contextId);
 247                         if (removed != 1)
 248                                 throw new Exception("Failed to delete shared context");
 249                         isUp = true;
 250                 } catch (Exception e) {
 251                         logger.error(EELFLoggerDelegate.errorLogger, "checkIfDatabaseUp failed", e);
 252                         isUp = false;
 253                 }
 254                 return isUp;
 255         }
 256
 257         private boolean checkClusterStatus() {
 258                 boolean isUp = false;
 259                 Session localSession = null;
 260                 try {
 261                         localSession = sessionFactory.openSession();
 262                         if (localSession != null) {
 263                                 // If all nodes are unhealthy in a cluster, this will throw an
 264                                 // exception
 265                                 String sql = "select * from mysql.user";
 266                                 Query query = localSession.createSQLQuery(sql);
 267                                 @SuppressWarnings("unchecked")
 268                                 List<String> queryList = query.list();
 269                                 if (queryList != null) {
 270                                         isUp = true;
 271                                 }
 272                         }
 273                 } catch (Exception e) {
 274                         logger.error(EELFLoggerDelegate.errorLogger, "checkClusterStatus failed", e);
 275                         if ((e.getCause() != null) && (e.getCause().getMessage() != null)) {
 276                                 logger.error(EELFLoggerDelegate.errorLogger,
 277                                                 "checkClusterStatus() exception cause", e.getCause());
 278                         }
 279                         isUp = false;
 280                 } finally {
 281                         if (localSession != null) {
 282                                 localSession.close();
 283                         }
 284                 }
 285                 return isUp;
 286         }
 287
 288         private boolean checkDatabasePermissions() {
 289                 boolean isUp = false;
 290                 Session localSession = null;
 291                 try {
 292                         localSession = sessionFactory.openSession();
 293                         if (localSession != null) {
 294                                 String sql = "SHOW GRANTS FOR CURRENT_USER";
 295                                 Query query = localSession.createSQLQuery(sql);
 296                                 @SuppressWarnings("unchecked")
 297                                 List<String> grantsList = query.list();
 298                                 for (String str : grantsList) {
 299                                         if ((str.toUpperCase().contains("ALL"))
 300                                                         || (str.toUpperCase().contains("DELETE") && str.toUpperCase().contains("SELECT")
 301                                                                         && str.toUpperCase().contains("UPDATE") && str.toUpperCase().contains("INSERT"))) {
 302                                                 isUp = true;
 303                                                 break;
 304                                         }
 305                                 }
 306                                 if (isUp == false) {
 307                                         logger.error(EELFLoggerDelegate.errorLogger,
 308                                                         "checkDatabaseAndPermissions() returning false.  SHOW GRANTS FOR CURRENT_USER being dumped:");
 309                                         for (String str : grantsList) {
 310                                                 logger.error(EELFLoggerDelegate.errorLogger, "grants output item = [" + str + "]");
 311                                         }
 312                                 }
 313                         }
 314                 } catch (Exception e) {
 315                         logger.error(EELFLoggerDelegate.errorLogger, "checkDatabasePermissions failed", e);
 316                         if ((e.getCause() != null) && (e.getCause().getMessage() != null)) {
 317                                 logger.error(EELFLoggerDelegate.errorLogger,
 318                                                 "checkDatabasePermissions() exception msg = ", e.getCause());
 319                         }
 320                         isUp = false;
 321                 } finally {
 322                         if (localSession != null) {
 323                                 localSession.close();
 324                         }
 325                 }
 326                 return isUp;
 327         }
 328
 329         private boolean checkIfUebUp() {
 330                 boolean uebUp = false;
 331                 try {
 332                         boolean isAvailable = epUebHelper.checkAvailability();
 333                         boolean messageCanBeSent = epUebHelper.MessageCanBeSentToTopic();
 334                         uebUp = (isAvailable && messageCanBeSent);
 335                 } catch (Exception e) {
 336                         logger.error(EELFLoggerDelegate.errorLogger, "checkIfUebUp failed", e);
 337                 }
 338                 return uebUp;
 339         }
 340
 341 }