ecomp-portal-BE-common/src/main/java/org/openecomp/portalapp/portal/listener/HealthMonitor.java

   1 /*-\r
   2  * ================================================================================\r
   3  * ECOMP Portal\r
   4  * ================================================================================\r
   5  * Copyright (C) 2017 AT&T Intellectual Property\r
   6  * ================================================================================\r
   7  * Licensed under the Apache License, Version 2.0 (the "License");\r
   8  * you may not use this file except in compliance with the License.\r
   9  * You may obtain a copy of the License at\r
  10  * \r
  11  *      http://www.apache.org/licenses/LICENSE-2.0\r
  12  * \r
  13  * Unless required by applicable law or agreed to in writing, software\r
  14  * distributed under the License is distributed on an "AS IS" BASIS,\r
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r
  16  * See the License for the specific language governing permissions and\r
  17  * limitations under the License.\r
  18  * ================================================================================\r
  19  */\r
  20 package org.openecomp.portalapp.portal.listener;\r
  21 \r
  22 import java.util.List;\r
  23 \r
  24 import javax.annotation.PostConstruct;\r
  25 import javax.annotation.PreDestroy;\r
  26 \r
  27 import org.hibernate.Query;\r
  28 import org.hibernate.Session;\r
  29 import org.hibernate.SessionFactory;\r
  30 import org.openecomp.portalapp.portal.logging.aop.EPMetricsLog;\r
  31 import org.openecomp.portalapp.portal.logging.format.EPAppMessagesEnum;\r
  32 import org.openecomp.portalapp.portal.logging.logic.EPLogUtil;\r
  33 import org.openecomp.portalapp.portal.ueb.EPUebHelper;\r
  34 import org.openecomp.portalapp.portal.utils.EPCommonSystemProperties;\r
  35 import org.openecomp.portalsdk.core.logging.logic.EELFLoggerDelegate;\r
  36 import org.openecomp.portalsdk.core.util.SystemProperties;\r
  37 import org.springframework.beans.factory.annotation.Autowired;\r
  38 import org.springframework.context.annotation.EnableAspectJAutoProxy;\r
  39 import org.springframework.transaction.annotation.Transactional;\r
  40 \r
  41 @Transactional\r
  42 @org.springframework.context.annotation.Configuration\r
  43 @EnableAspectJAutoProxy\r
  44 @EPMetricsLog\r
  45 public class HealthMonitor {\r
  46 \r
  47         @Autowired\r
  48         private SessionFactory sessionFactory;\r
  49 \r
  50         @Autowired\r
  51         private EPUebHelper epUebHelper;\r
  52 \r
  53         private static boolean databaseUp;\r
  54         private static boolean uebUp;\r
  55         private static boolean frontEndUp;\r
  56         private static boolean backEndUp;\r
  57         private static boolean dbClusterStatusOk;\r
  58         private static boolean dbPermissionsOk;\r
  59         public static boolean isSuspended = false;\r
  60 \r
  61         Thread healthMonitorThread;\r
  62 \r
  63         EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(HealthMonitor.class);\r
  64 \r
  65         public HealthMonitor() {\r
  66 \r
  67         }\r
  68 \r
  69         public static boolean isDatabaseUp() {\r
  70                 return databaseUp;\r
  71         }\r
  72 \r
  73         public static boolean isClusterStatusOk() {\r
  74                 return dbClusterStatusOk;\r
  75         }\r
  76 \r
  77         public static boolean isDatabasePermissionsOk() {\r
  78                 return dbPermissionsOk;\r
  79         }\r
  80 \r
  81         public static boolean isUebUp() {\r
  82                 return uebUp;\r
  83         }\r
  84 \r
  85         public static boolean isFrontEndUp() {\r
  86                 return frontEndUp;\r
  87         }\r
  88 \r
  89         public static boolean isBackEndUp() {\r
  90                 return backEndUp;\r
  91         }\r
  92 \r
  93         private void monitorEPHealth() throws InterruptedException {\r
  94 \r
  95                 int numIntervalsDatabaseHasBeenDown = 0;\r
  96                 int numIntervalsClusterNotHealthy = 0;\r
  97                 int numIntervalsDatabasePermissionsIncorrect = 0;\r
  98                 int numIntervalsUebHasBeenDown = 0;\r
  99 \r
 100                 logger.debug(EELFLoggerDelegate.debugLogger, "==> Health Monitor thread started");\r
 101 \r
 102                 long sleepInterval = (Long\r
 103                                 .valueOf(SystemProperties.getProperty(EPCommonSystemProperties.HEALTH_POLL_INTERVAL_SECONDS)) * 1000);\r
 104                 long numIntervalsBetweenAlerts = Long\r
 105                                 .valueOf(SystemProperties.getProperty(EPCommonSystemProperties.HEALTHFAIL_ALERT_EVERY_X_INTERVALS));\r
 106                 logger.debug(EELFLoggerDelegate.debugLogger,\r
 107                                 "Polling health every " + sleepInterval + " milliseconds. Alerting every "\r
 108                                                 + (sleepInterval * numIntervalsBetweenAlerts) / 1000 + " seconds when component remains down.");\r
 109 \r
 110                 while (true) {\r
 111                         //\r
 112                         // Get DB status. If down, signal alert once every X intervals.\r
 113                         //\r
 114                         databaseUp = this.checkIfDatabaseUp();\r
 115                         if (databaseUp == false) {\r
 116 \r
 117                                 if ((numIntervalsDatabaseHasBeenDown % numIntervalsBetweenAlerts) == 0) {\r
 118                                         // Write a Log entry that will generate an alert\r
 119                                         EPLogUtil.logEcompError(logger, EPAppMessagesEnum.BeHealthCheckMySqlError);\r
 120                                         logger.debug(EELFLoggerDelegate.debugLogger,\r
 121                                                         "Database down, logging to error log to trigger alert.");\r
 122                                         numIntervalsDatabaseHasBeenDown++;\r
 123                                 } else {\r
 124                                         numIntervalsDatabaseHasBeenDown = 0;\r
 125                                 }\r
 126                         }\r
 127 \r
 128                         dbClusterStatusOk = this.checkClusterStatus();\r
 129                         if (dbClusterStatusOk == false) {\r
 130 \r
 131                                 if ((numIntervalsClusterNotHealthy % numIntervalsBetweenAlerts) == 0) {\r
 132                                         EPLogUtil.logEcompError(logger, EPAppMessagesEnum.BeHealthCheckMySqlError);\r
 133                                         logger.debug(EELFLoggerDelegate.debugLogger,\r
 134                                                         "Cluster nodes appear to be down, logging to error log to trigger alert.");\r
 135                                         numIntervalsClusterNotHealthy++;\r
 136                                 } else {\r
 137                                         numIntervalsClusterNotHealthy = 0;\r
 138                                 }\r
 139                         }\r
 140 \r
 141                         dbPermissionsOk = this.checkDatabaseAndPermissions();\r
 142                         if (dbPermissionsOk == false) {\r
 143 \r
 144                                 if ((numIntervalsDatabasePermissionsIncorrect % numIntervalsBetweenAlerts) == 0) {\r
 145                                         EPLogUtil.logEcompError(logger, EPAppMessagesEnum.BeHealthCheckMySqlError);\r
 146                                         logger.debug(EELFLoggerDelegate.debugLogger,\r
 147                                                         "Database permissions don't seem correct, logging to error log to trigger alert.");\r
 148                                         numIntervalsDatabasePermissionsIncorrect++;\r
 149                                 } else {\r
 150                                         numIntervalsDatabasePermissionsIncorrect = 0;\r
 151                                 }\r
 152                         }\r
 153 \r
 154                         //\r
 155                         // Get UEB status. Publish a bogus message to EP inbox, if 200 OK\r
 156                         // returned, status is Up.\r
 157                         // If down, signal alert once every X intervals.\r
 158                         // EP will ignore this bogus message.\r
 159                         //\r
 160                         uebUp = this.checkIfUebUp();\r
 161                         if (uebUp == false) {\r
 162 \r
 163                                 if ((numIntervalsUebHasBeenDown % numIntervalsBetweenAlerts) == 0) {\r
 164                                         // Write a Log entry that will generate an alert\r
 165                                         EPLogUtil.logEcompError(logger, EPAppMessagesEnum.BeHealthCheckUebClusterError);\r
 166                                         logger.debug(EELFLoggerDelegate.debugLogger, "UEB down, logging to error log to trigger alert");\r
 167                                         numIntervalsUebHasBeenDown++;\r
 168                                 } else {\r
 169                                         numIntervalsUebHasBeenDown = 0;\r
 170                                 }\r
 171                         }\r
 172 \r
 173                         // The front end should be up because the API is called through\r
 174                         // proxy front end server.\r
 175                         frontEndUp = true;\r
 176 \r
 177                         // If the rest API called, the backend is always up\r
 178                         backEndUp = true;\r
 179 \r
 180                         //\r
 181                         // future nice to have...get Partner status\r
 182                         //\r
 183                         // For all apps exposing a rest url, query one of the rest\r
 184                         // urls(/roles?) and manage a list\r
 185                         // of app name/status. We might not return back a non 200 OK in\r
 186                         // health check, but we\r
 187                         // could return information in the json content of a health check.\r
 188                         //\r
 189 \r
 190                         //\r
 191                         // Get DB status. If down, signal alert once every X intervals.\r
 192                         //\r
 193                         if (Thread.interrupted()) {\r
 194                                 logger.info(EELFLoggerDelegate.errorLogger, "==> UebMainHandler exiting");\r
 195                                 break;\r
 196                         }\r
 197 \r
 198                         try {\r
 199                                 Thread.sleep(sleepInterval);\r
 200                         } catch (InterruptedException e) {\r
 201                                 logger.error(EELFLoggerDelegate.errorLogger, "monitorEPHealth interrupted", e);\r
 202                                 Thread.currentThread().interrupt();\r
 203                         }\r
 204                 }\r
 205         }\r
 206 \r
 207         @PostConstruct\r
 208         public void initHealthMonitor() {\r
 209 \r
 210                 healthMonitorThread = new Thread("EP HealthMonitor thread") {\r
 211                         public void run() {\r
 212                                 try {\r
 213                                         monitorEPHealth();\r
 214                                 } catch (InterruptedException e) {\r
 215                                         logger.debug(EELFLoggerDelegate.debugLogger, "healthMonitorThread interrupted", e);\r
 216                                 } catch (Exception e) {\r
 217                                         logger.error(EELFLoggerDelegate.errorLogger, "healthMonitorThread failed", e);\r
 218                                 }\r
 219                         }\r
 220                 };\r
 221                 if (healthMonitorThread != null) {\r
 222                         healthMonitorThread.start();\r
 223                 }\r
 224         }\r
 225 \r
 226         @PreDestroy\r
 227         public void closeHealthMonitor() {\r
 228                 this.healthMonitorThread.interrupt();\r
 229         }\r
 230 \r
 231         private boolean checkIfDatabaseUp() {\r
 232 \r
 233                 boolean isUp = false;\r
 234 \r
 235                 Session localSession = null;\r
 236 \r
 237                 try {\r
 238                         localSession = sessionFactory.openSession();\r
 239 \r
 240                         if (localSession != null) {\r
 241 \r
 242                                 String sql = "select app_name from fn_app where app_id=1";\r
 243                                 Query query = localSession.createSQLQuery(sql);\r
 244                                 @SuppressWarnings("unchecked")\r
 245                                 List<String> queryList = query.list();\r
 246                                 if (queryList != null) {\r
 247                                         isUp = true;\r
 248                                 }\r
 249                                 localSession.close();\r
 250                         }\r
 251                 } catch (Exception e) {\r
 252                         logger.debug(EELFLoggerDelegate.debugLogger, "checkIfDatabaseUp failed", e);\r
 253                         isUp = false;\r
 254                 }\r
 255 \r
 256                 return isUp;\r
 257         }\r
 258 \r
 259         private boolean checkClusterStatus() {\r
 260 \r
 261                 boolean isUp = false;\r
 262 \r
 263                 Session localSession = null;\r
 264 \r
 265                 try {\r
 266                         localSession = sessionFactory.openSession();\r
 267                         if (localSession != null) {\r
 268                                 // If all nodes are unhealthy in a cluster, this will throw an\r
 269                                 // exception\r
 270                                 String sql = "select * from mysql.user";\r
 271                                 Query query = localSession.createSQLQuery(sql);\r
 272                                 @SuppressWarnings("unchecked")\r
 273                                 List<String> queryList = query.list();\r
 274                                 if (queryList != null) {\r
 275                                         isUp = true;\r
 276                                 }\r
 277                         }\r
 278                 } catch (Exception e) {\r
 279                         logger.error(EELFLoggerDelegate.errorLogger, "checkClusterStatus failed", e);\r
 280                         if ((e.getCause() != null) && (e.getCause().getMessage() != null)) {\r
 281                                 logger.error(EELFLoggerDelegate.errorLogger,\r
 282                                                 "checkClusterStatus() exception msg = " + e.getCause().getMessage());\r
 283                         }\r
 284                         isUp = false;\r
 285                 } finally {\r
 286                         if (localSession != null) {\r
 287                                 localSession.close();\r
 288                         }\r
 289                 }\r
 290 \r
 291                 return isUp;\r
 292 \r
 293         }\r
 294 \r
 295         private boolean checkDatabaseAndPermissions() {\r
 296 \r
 297                 boolean isUp = false;\r
 298 \r
 299                 Session localSession = null;\r
 300 \r
 301                 try {\r
 302                         localSession = sessionFactory.openSession();\r
 303                         if (localSession != null) {\r
 304                                 String sql = "SHOW GRANTS FOR CURRENT_USER";\r
 305                                 Query query = localSession.createSQLQuery(sql);\r
 306                                 @SuppressWarnings("unchecked")\r
 307                                 List<String> grantsList = query.list();\r
 308                                 for (String str : grantsList) {\r
 309                                         if ((str.toUpperCase().contains("ALL"))\r
 310                                                         || (str.toUpperCase().contains("DELETE") && str.toUpperCase().contains("SELECT")\r
 311                                                                         && str.toUpperCase().contains("UPDATE") && str.toUpperCase().contains("INSERT"))) {\r
 312                                                 isUp = true;\r
 313                                                 break;\r
 314                                         }\r
 315                                 }\r
 316                                 if (isUp == false) {\r
 317                                         logger.error(EELFLoggerDelegate.errorLogger,\r
 318                                                         "checkDatabaseAndPermissions() returning false.  SHOW GRANTS FOR CURRENT_USER being dumped:");\r
 319                                         for (String str : grantsList) {\r
 320                                                 logger.error(EELFLoggerDelegate.errorLogger, "grants output item = [" + str + "]");\r
 321                                         }\r
 322                                 }\r
 323                         }\r
 324                 } catch (Exception e) {\r
 325                         logger.error(EELFLoggerDelegate.errorLogger, "checkDatabaseAndPermissions failed", e);\r
 326                         if ((e.getCause() != null) && (e.getCause().getMessage() != null)) {\r
 327                                 logger.error(EELFLoggerDelegate.errorLogger,\r
 328                                                 "checkDatabaseAndPermissions() exception msg = " + e.getCause().getMessage());\r
 329                         }\r
 330                         isUp = false;\r
 331                 } finally {\r
 332                         if (localSession != null) {\r
 333                                 localSession.close();\r
 334                         }\r
 335                 }\r
 336 \r
 337                 return isUp;\r
 338 \r
 339         }\r
 340 \r
 341         private boolean checkIfUebUp() {\r
 342                 boolean uebUp = false;\r
 343                 try {\r
 344                         boolean isAvailable = epUebHelper.checkAvailability();\r
 345                         boolean messageCanBeSent = epUebHelper.MessageCanBeSentToTopic();\r
 346                         uebUp = (isAvailable && messageCanBeSent);\r
 347                 } catch (Exception e) {\r
 348                         logger.error(EELFLoggerDelegate.errorLogger, "checkIfUebUp failed", e);\r
 349                 }\r
 350 \r
 351                 return uebUp;\r
 352 \r
 353         }\r
 354 \r
 355 }\r