ecomp-portal-BE/src/main/java/org/openecomp/portalapp/portal/listener/HealthMonitor.java

   1 /*-
   2  * ================================================================================
   3  * eCOMP Portal
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ================================================================================
  19  */
  20 package org.openecomp.portalapp.portal.listener;
  21
  22 import java.util.List;
  23
  24 import javax.annotation.PostConstruct;
  25 import javax.annotation.PreDestroy;
  26
  27 import org.hibernate.Query;
  28 import org.hibernate.Session;
  29 import org.hibernate.SessionFactory;
  30 import org.openecomp.portalapp.portal.logging.aop.EPMetricsLog;
  31 import org.openecomp.portalapp.portal.logging.format.EPAppMessagesEnum;
  32 import org.openecomp.portalapp.portal.logging.logic.EPLogUtil;
  33 import org.openecomp.portalapp.portal.ueb.EPUebHelper;
  34 import org.openecomp.portalapp.portal.utils.EPSystemProperties;
  35 import org.openecomp.portalapp.portal.utils.EcompPortalUtils;
  36 import org.openecomp.portalsdk.core.logging.logic.EELFLoggerDelegate;
  37 import org.openecomp.portalsdk.core.util.SystemProperties;
  38 import org.springframework.beans.factory.annotation.Autowired;
  39 import org.springframework.context.annotation.EnableAspectJAutoProxy;
  40 import org.springframework.transaction.annotation.Transactional;
  41
  42 @Transactional
  43 @org.springframework.context.annotation.Configuration
  44 @EnableAspectJAutoProxy
  45 @EPMetricsLog
  46 public class HealthMonitor {
  47
  48         @Autowired
  49         private SessionFactory sessionFactory;
  50
  51         @Autowired
  52         private EPUebHelper epUebHelper;
  53
  54         private static boolean databaseUp;
  55         private static boolean uebUp;
  56         private static boolean frontEndUp;
  57         private static boolean backEndUp;
  58         private static boolean dbClusterStatusOk;
  59         private static boolean dbPermissionsOk;
  60         public static boolean isSuspended = false;
  61
  62         Thread healthMonitorThread;
  63
  64         EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(HealthMonitor.class);
  65
  66         public HealthMonitor() {
  67
  68         }
  69
  70         public static boolean isDatabaseUp() {
  71                 return databaseUp;
  72         }
  73
  74         public static boolean isClusterStatusOk() {
  75                 return dbClusterStatusOk;
  76         }
  77
  78         public static boolean isDatabasePermissionsOk() {
  79                 return dbPermissionsOk;
  80         }
  81
  82         public static boolean isUebUp() {
  83                 return uebUp;
  84         }
  85
  86         public static boolean isFrontEndUp() {
  87                 return frontEndUp;
  88         }
  89
  90         public static boolean isBackEndUp() {
  91                 return backEndUp;
  92         }
  93
  94         private void monitorEPHealth() throws InterruptedException{
  95
  96                 int numIntervalsDatabaseHasBeenDown = 0;
  97                 int numIntervalsClusterNotHealthy = 0;
  98                 int numIntervalsDatabasePermissionsIncorrect = 0;
  99                 int numIntervalsUebHasBeenDown = 0;
 100
 101                 logger.debug(EELFLoggerDelegate.debugLogger, "==> Health Monitor thread started");
 102
 103                 long sleepInterval = (Long.valueOf(SystemProperties.getProperty(EPSystemProperties.HEALTH_POLL_INTERVAL_SECONDS)) * 1000);
 104                 long numIntervalsBetweenAlerts = Long.valueOf(SystemProperties.getProperty(EPSystemProperties.HEALTHFAIL_ALERT_EVERY_X_INTERVALS));
 105                 logger.debug(EELFLoggerDelegate.debugLogger, "Polling health every " + sleepInterval + " milliseconds. Alerting every "
 106                                   + (sleepInterval * numIntervalsBetweenAlerts)/1000 + " seconds when component remains down.");
 107
 108                 while (true)
 109                 {
 110                     //
 111                 //  Get DB status.  If down, signal alert once every X intervals.
 112                 //
 113                 databaseUp = this.checkIfDatabaseUp();
 114                     if (databaseUp == false) {
 115
 116                         if ((numIntervalsDatabaseHasBeenDown % numIntervalsBetweenAlerts) == 0){
 117                                     // Write a Log entry that will generate an alert
 118                                 EPLogUtil.logEcompError(EPAppMessagesEnum.BeHealthCheckMySqlError);
 119                                 logger.debug(EELFLoggerDelegate.debugLogger, "Database down, logging to error log to trigger alert.");
 120                             numIntervalsDatabaseHasBeenDown++;
 121                         }
 122                         else {
 123                                 numIntervalsDatabaseHasBeenDown = 0;
 124                         }
 125                     }
 126
 127                 dbClusterStatusOk = this.checkClusterStatus();
 128                     if (dbClusterStatusOk == false) {
 129
 130                         if ((numIntervalsClusterNotHealthy % numIntervalsBetweenAlerts) == 0){
 131                                 EPLogUtil.logEcompError(EPAppMessagesEnum.BeHealthCheckMySqlError);
 132                                 logger.debug(EELFLoggerDelegate.debugLogger, "Cluster nodes appear to be down, logging to error log to trigger alert.");
 133                                 numIntervalsClusterNotHealthy++;
 134                         }
 135                         else {
 136                                 numIntervalsClusterNotHealthy = 0;
 137                         }
 138                     }
 139
 140                 dbPermissionsOk = this.checkDatabaseAndPermissions();
 141                     if (dbPermissionsOk == false) {
 142
 143                         if ((numIntervalsDatabasePermissionsIncorrect % numIntervalsBetweenAlerts) == 0){
 144                                 EPLogUtil.logEcompError(EPAppMessagesEnum.BeHealthCheckMySqlError);
 145                                 logger.debug(EELFLoggerDelegate.debugLogger, "Database permissions don't seem correct, logging to error log to trigger alert.");
 146                                 numIntervalsDatabasePermissionsIncorrect++;
 147                         }
 148                         else {
 149                                 numIntervalsDatabasePermissionsIncorrect = 0;
 150                         }
 151                     }
 152
 153                     //
 154                     //  Get UEB status.  Publish a bogus message to EP inbox, if 200 OK returned, status is Up.
 155                     //  If down, signal alert once every X intervals.
 156                     //  EP will ignore this bogus message.
 157                     //
 158                     uebUp = this.checkIfUebUp();
 159             if (uebUp == false) {
 160
 161                 if ((numIntervalsUebHasBeenDown % numIntervalsBetweenAlerts) == 0){
 162                                     // Write a Log entry that will generate an alert
 163                         EPLogUtil.logEcompError(EPAppMessagesEnum.BeHealthCheckUebClusterError);
 164                                 logger.debug(EELFLoggerDelegate.debugLogger, "UEB down, logging to error log to trigger alert");
 165                             numIntervalsUebHasBeenDown++;
 166                         }
 167                         else {
 168                                 numIntervalsUebHasBeenDown = 0;
 169                         }
 170                     }
 171
 172             //The front end should be up because the API is called through proxy front end server.
 173             frontEndUp = true;
 174
 175
 176                     //If the rest API called, the backend  is always up
 177                     backEndUp = true;
 178
 179                     //
 180                     //  future nice to have...get Partner status
 181                     //
 182                     //  For all apps exposing a rest url, query one of the rest urls(/roles?) and manage a list
 183                     //  of app name/status.  We might not return back a non 200 OK in health check, but we
 184             //  could return information in the json content of a health check.
 185                     //
 186
 187             //
 188                 //  Get DB status.  If down, signal alert once every X intervals.
 189                 //
 190                     Thread.sleep(sleepInterval);
 191                 }
 192         }
 193
 194         @PostConstruct
 195         public void initHealthMonitor() {
 196
 197                 healthMonitorThread = new Thread("EP HealthMonitor thread") {
 198                     public void run(){
 199                         try {
 200                                         monitorEPHealth();
 201                                 } catch (InterruptedException e) {
 202                                         logger.debug(EELFLoggerDelegate.debugLogger, "<== Health Monitor thread exiting..." + EcompPortalUtils.getStackTrace(e));
 203                                 } catch(Exception e) {
 204                                         logger.error(EELFLoggerDelegate.errorLogger, "Exception occurred in Healthcheck monitor thread." + EcompPortalUtils.getStackTrace(e));
 205                                 }
 206                     }
 207                 };
 208                 if (healthMonitorThread != null) {
 209                     healthMonitorThread.start();
 210                 }
 211         }
 212
 213         @PreDestroy
 214         public void closeHealthMonitor()
 215         {
 216                 this.healthMonitorThread.interrupt();
 217         }
 218
 219
 220     private boolean checkIfDatabaseUp() {
 221
 222                 boolean isUp = false;
 223
 224                 Session localSession = null;
 225
 226                 try {
 227                         localSession = sessionFactory.openSession();
 228
 229                         if (localSession != null) {
 230
 231                                 String sql = "select app_name from fn_app where app_id=1";
 232                         Query query = localSession.createSQLQuery(sql);
 233                         @SuppressWarnings("unchecked")
 234                             List <String>queryList = query.list();
 235                         if (queryList != null) {
 236                             isUp = true;
 237                         }
 238                         localSession.close();
 239                     }
 240                 }
 241                 catch (Exception e) {
 242                         logger.error(EELFLoggerDelegate.errorLogger, EcompPortalUtils.getStackTrace(e));
 243                         isUp = false;
 244                 }
 245
 246                 return isUp;
 247         }
 248
 249
 250     private boolean checkClusterStatus() {
 251
 252                 boolean isUp = false;
 253
 254                 Session localSession = null;
 255
 256                 try {
 257                         localSession = sessionFactory.openSession();
 258                         if (localSession != null) {
 259                             /////////////////////////////////////////////////////////////////////////
 260                             // If all nodes are unhealthy in a cluster, this will throw an exception
 261                             /////////////////////////////////////////////////////////////////////////
 262                         String sql = "select * from mysql.user";
 263                         Query query = localSession.createSQLQuery(sql);
 264                         @SuppressWarnings("unchecked")
 265                             List <String>queryList = query.list();
 266                         if (queryList != null) {
 267                             isUp = true;
 268                         }
 269                         }
 270                 }
 271                 catch (Exception e) {
 272                         logger.error(EELFLoggerDelegate.errorLogger, "checkClusterStatus() exception msg = " + e.getMessage());
 273                         if ((e.getCause() != null) && (e.getCause().getMessage() != null)) {
 274                             logger.error(EELFLoggerDelegate.errorLogger, "checkClusterStatus() exception msg = " + e.getCause().getMessage());
 275                         }
 276                         logger.error(EELFLoggerDelegate.errorLogger, "Exception inside checkClusterStatus() exception = " + EcompPortalUtils.getStackTrace(e));
 277                         isUp = false;
 278                 }
 279                 finally {
 280                         if (localSession != null) {
 281                                 localSession.close();
 282                         }
 283                 }
 284
 285                 return isUp;
 286
 287         }
 288
 289
 290     private boolean checkDatabaseAndPermissions() {
 291
 292                 boolean isUp = false;
 293
 294                 Session localSession = null;
 295
 296                 try {
 297                         localSession = sessionFactory.openSession();
 298                         if (localSession != null) {
 299                         String sql = "SHOW GRANTS FOR CURRENT_USER";
 300                         Query query = localSession.createSQLQuery(sql);
 301                         @SuppressWarnings("unchecked")
 302                         List<String> grantsList = query.list();
 303                         for (String str : grantsList) {
 304                             if ((str.toUpperCase().contains("ALL"))
 305                                         ||
 306                                 (str.toUpperCase().contains("DELETE") &&
 307                                  str.toUpperCase().contains("SELECT") &&
 308                                  str.toUpperCase().contains("UPDATE") &&
 309                                  str.toUpperCase().contains("INSERT"))) {
 310                                     isUp = true;
 311                                     break;
 312                             }
 313                         }
 314                         if (isUp == false) {
 315                             logger.error(EELFLoggerDelegate.errorLogger, "checkDatabaseAndPermissions() returning false.  SHOW GRANTS FOR CURRENT_USER being dumped:");
 316                             for (String str : grantsList) {
 317                                     logger.error(EELFLoggerDelegate.errorLogger, "grants output item = [" + str + "]");
 318                                 }
 319                         }
 320                         }
 321                 }
 322                 catch (Exception e) {
 323                         logger.error(EELFLoggerDelegate.errorLogger, "checkDatabaseAndPermissions() exception msg = " + e.getMessage());
 324                         if ((e.getCause() != null) && (e.getCause().getMessage() != null)) {
 325                             logger.error(EELFLoggerDelegate.errorLogger, "checkDatabaseAndPermissions() exception msg = " + e.getCause().getMessage());
 326                         }
 327                         logger.error(EELFLoggerDelegate.errorLogger, "Exception inside checkDatabaseAndPermissions() exception = " + EcompPortalUtils.getStackTrace(e));
 328                         isUp = false;
 329                 }
 330                 finally {
 331                         if (localSession != null) {
 332                                 localSession.close();
 333                         }
 334                 }
 335
 336                 return isUp;
 337
 338         }
 339
 340
 341         private boolean checkIfUebUp() {
 342                 boolean uebUp = false;
 343                 try {
 344                     boolean isAvailable = epUebHelper.checkAvailability();
 345                     boolean messageCanBeSent = epUebHelper.MessageCanBeSentToTopic();
 346                     uebUp = (isAvailable && messageCanBeSent);
 347                 }
 348                 catch (Exception e) {
 349                         logger.error(EELFLoggerDelegate.errorLogger, "Exception inside CheckIfUebUp exception = " + EcompPortalUtils.getStackTrace(e));
 350                 }
 351
 352                 return uebUp;
 353
 354
 355         }
 356
 357         /*
 358         private boolean checkIfFeUp() {
 359                 boolean frontenndUp = false;
 360                 try {
 361                         String url = SystemProperties.getProperty(EPSystemProperties.FE_URL);
 362                     if(StringUtils.isEmpty(url))
 363                     {
 364                                 logger.debug(EELFLoggerDelegate.debugLogger, "The front end URL is empty. Cannot check the status");
 365                         return frontenndUp;
 366                     }
 367
 368                     url = url.replace("index.html", "app/healthCheck.json");
 369
 370                     URL frontEndURL = new URL(url);
 371
 372                     HttpURLConnection con = (HttpURLConnection) frontEndURL.openConnection();
 373
 374                     // optional default is GET
 375                     con.setRequestMethod("GET");
 376
 377                     int responseCode = con.getResponseCode();
 378                     logger.debug(EELFLoggerDelegate.debugLogger, "Fronend response code : " + responseCode);
 379
 380                     if(responseCode == 200)
 381                     {
 382                         frontenndUp = true;
 383                     }
 384
 385                 }
 386                 catch(Exception e){
 387                         logger.error(EELFLoggerDelegate.errorLogger, "Exception occurred while trying to access font end" + EcompPortalUtils.getStackTrace(e));
 388             }
 389
 390                 return frontenndUp;
 391         }
 392         */
 393 }