catalog-be/src/main/java/org/openecomp/sdc/be/components/impl/HealthCheckBusinessLogic.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.be.components.impl;
  22
  23 import java.util.ArrayList;
  24 import java.util.List;
  25 import java.util.Map;
  26 import java.util.Map.Entry;
  27 import java.util.concurrent.Executors;
  28 import java.util.concurrent.ScheduledExecutorService;
  29 import java.util.concurrent.ScheduledFuture;
  30 import java.util.concurrent.ThreadFactory;
  31 import java.util.concurrent.TimeUnit;
  32 import java.util.stream.Collectors;
  33
  34 import javax.annotation.PostConstruct;
  35 import javax.annotation.PreDestroy;
  36 import javax.annotation.Resource;
  37 import javax.servlet.ServletContext;
  38
  39 import org.openecomp.sdc.be.components.distribution.engine.DistributionEngineClusterHealth;
  40 import org.openecomp.sdc.be.components.distribution.engine.UebHealthCheckCall;
  41 import org.openecomp.sdc.be.config.BeEcompErrorManager;
  42 import org.openecomp.sdc.be.config.BeEcompErrorManager.ErrorSeverity;
  43 import org.openecomp.sdc.be.dao.api.IEsHealthCheckDao;
  44 import org.openecomp.sdc.be.dao.titan.TitanGenericDao;
  45 import org.openecomp.sdc.be.impl.WebAppContextWrapper;
  46 import org.openecomp.sdc.be.switchover.detector.SwitchoverDetector;
  47 import org.openecomp.sdc.common.api.Constants;
  48 import org.openecomp.sdc.common.api.HealthCheckInfo;
  49 import org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckComponent;
  50 import org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus;
  51 import org.openecomp.sdc.common.impl.ExternalConfiguration;
  52 import org.slf4j.Logger;
  53 import org.slf4j.LoggerFactory;
  54 import org.springframework.beans.factory.annotation.Autowired;
  55 import org.springframework.stereotype.Component;
  56 import org.springframework.web.context.WebApplicationContext;
  57
  58 @Component("healthCheckBusinessLogic")
  59 public class HealthCheckBusinessLogic {
  60
  61         protected static String BE_HEALTH_LOG_CONTEXT = "be.healthcheck";
  62
  63         private static Logger healthLogger = LoggerFactory.getLogger(BE_HEALTH_LOG_CONTEXT);
  64
  65         private static final String BE_HEALTH_CHECK_STR = "beHealthCheck";
  66
  67         @Resource
  68         private TitanGenericDao titanGenericDao;
  69
  70         @Resource
  71         private IEsHealthCheckDao esHealthCheckDao;
  72
  73         @Resource
  74         private DistributionEngineClusterHealth distributionEngineClusterHealth;
  75
  76         @Autowired
  77         private SwitchoverDetector switchoverDetector;
  78
  79         private static Logger log = LoggerFactory.getLogger(HealthCheckBusinessLogic.class.getName());
  80
  81         private volatile List<HealthCheckInfo> lastBeHealthCheckInfos = null;
  82
  83         // private static volatile HealthCheckBusinessLogic instance;
  84         //
  85         public HealthCheckBusinessLogic() {
  86
  87         }
  88
  89         private ScheduledFuture<?> scheduledFuture = null;
  90
  91         ScheduledExecutorService healthCheckScheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
  92                 @Override
  93                 public Thread newThread(Runnable r) {
  94                         return new Thread(r, "BE-Health-Check-Task");
  95                 }
  96         });
  97
  98         HealthCheckScheduledTask healthCheckScheduledTask = null;
  99
 100         @PostConstruct
 101         public void init() {
 102
 103                 lastBeHealthCheckInfos = getBeHealthCheckInfos();
 104
 105                 log.debug("After initializing lastBeHealthCheckInfos :{}", lastBeHealthCheckInfos);
 106
 107                 healthCheckScheduledTask = new HealthCheckScheduledTask();
 108
 109                 if (this.scheduledFuture == null) {
 110                         this.scheduledFuture = this.healthCheckScheduler.scheduleAtFixedRate(healthCheckScheduledTask, 0, 3, TimeUnit.SECONDS);
 111                 }
 112
 113         }
 114
 115         //
 116         // public static HealthCheckBusinessLogic getInstance(){
 117         //// if (instance == null){
 118         //// instance = init();
 119         //// }
 120         // return instance;
 121         // }
 122
 123         // private synchronized static HealthCheckBusinessLogic init() {
 124         // if (instance == null){
 125         // instance = new HealthCheckBusinessLogic();
 126         // }
 127         // return instance;
 128         // }
 129
 130         private List<HealthCheckInfo> getBeHealthCheckInfos(ServletContext servletContext) {
 131
 132                 List<HealthCheckInfo> healthCheckInfos = new ArrayList<HealthCheckInfo>();
 133
 134                 // BE
 135                 getBeHealthCheck(servletContext, healthCheckInfos);
 136
 137                 // ES
 138                 getEsHealthCheck(servletContext, healthCheckInfos);
 139
 140                 // Titan
 141                 getTitanHealthCheck(servletContext, healthCheckInfos);
 142
 143                 // Distribution Engine
 144                 getDistributionEngineCheck(servletContext, healthCheckInfos);
 145
 146                 return healthCheckInfos;
 147         }
 148
 149         private List<HealthCheckInfo> getBeHealthCheck(ServletContext servletContext, List<HealthCheckInfo> healthCheckInfos) {
 150                 String appVersion = ExternalConfiguration.getAppVersion();
 151                 String description = "OK";
 152                 healthCheckInfos.add(new HealthCheckInfo(HealthCheckComponent.BE, HealthCheckStatus.UP, appVersion, description));
 153                 return healthCheckInfos;
 154         }
 155
 156         public List<HealthCheckInfo> getTitanHealthCheck(ServletContext servletContext, List<HealthCheckInfo> healthCheckInfos) {
 157                 // Titan health check and version
 158                 TitanGenericDao titanStatusDao = (TitanGenericDao) getDao(servletContext, TitanGenericDao.class);
 159                 String description;
 160                 boolean isTitanUp;
 161
 162                 try {
 163                         isTitanUp = titanStatusDao.isGraphOpen();
 164                 } catch (Exception e) {
 165                         description = "Titan error: " + e.getMessage();
 166                         healthCheckInfos.add(new HealthCheckInfo(HealthCheckComponent.TITAN, HealthCheckStatus.DOWN, null, description));
 167                         return healthCheckInfos;
 168                 }
 169                 if (isTitanUp) {
 170                         description = "OK";
 171                         healthCheckInfos.add(new HealthCheckInfo(HealthCheckComponent.TITAN, HealthCheckStatus.UP, null, description));
 172                 } else {
 173                         description = "Titan graph is down";
 174                         healthCheckInfos.add(new HealthCheckInfo(HealthCheckComponent.TITAN, HealthCheckStatus.DOWN, null, description));
 175                 }
 176                 return healthCheckInfos;
 177         }
 178
 179         public List<HealthCheckInfo> getEsHealthCheck(ServletContext servletContext, List<HealthCheckInfo> healthCheckInfos) {
 180
 181                 // ES health check and version
 182                 IEsHealthCheckDao esStatusDao = (IEsHealthCheckDao) getDao(servletContext, IEsHealthCheckDao.class);
 183                 HealthCheckStatus healthCheckStatus;
 184                 String description;
 185
 186                 try {
 187                         healthCheckStatus = esStatusDao.getClusterHealthStatus();
 188                 } catch (Exception e) {
 189                         healthCheckStatus = HealthCheckStatus.DOWN;
 190                         description = "ES cluster error: " + e.getMessage();
 191                         healthCheckInfos.add(new HealthCheckInfo(HealthCheckComponent.ES, healthCheckStatus, null, description));
 192                         return healthCheckInfos;
 193                 }
 194                 if (healthCheckStatus.equals(HealthCheckStatus.DOWN)) {
 195                         description = "ES cluster is down";
 196                 } else {
 197                         description = "OK";
 198                 }
 199                 healthCheckInfos.add(new HealthCheckInfo(HealthCheckComponent.ES, healthCheckStatus, null, description));
 200                 return healthCheckInfos;
 201         }
 202
 203         public Object getDao(ServletContext servletContext, Class<?> clazz) {
 204                 WebAppContextWrapper webApplicationContextWrapper = (WebAppContextWrapper) servletContext.getAttribute(Constants.WEB_APPLICATION_CONTEXT_WRAPPER_ATTR);
 205
 206                 WebApplicationContext webApplicationContext = webApplicationContextWrapper.getWebAppContext(servletContext);
 207
 208                 return webApplicationContext.getBean(clazz);
 209         }
 210
 211         private void getDistributionEngineCheck(ServletContext servletContext, List<HealthCheckInfo> healthCheckInfos) {
 212
 213                 DistributionEngineClusterHealth deDao = (DistributionEngineClusterHealth) getDao(servletContext, DistributionEngineClusterHealth.class);
 214                 HealthCheckInfo healthCheckInfo = deDao.getHealthCheckInfo();
 215
 216                 healthCheckInfos.add(healthCheckInfo);
 217
 218         }
 219
 220         public boolean isDistributionEngineUp(ServletContext servletContext) {
 221
 222                 DistributionEngineClusterHealth deDao = (DistributionEngineClusterHealth) getDao(servletContext, DistributionEngineClusterHealth.class);
 223                 HealthCheckInfo healthCheckInfo = deDao.getHealthCheckInfo();
 224                 if (healthCheckInfo.getHealthCheckStatus().equals(HealthCheckStatus.DOWN)) {
 225                         return false;
 226                 }
 227                 return true;
 228         }
 229
 230         public List<HealthCheckInfo> getBeHealthCheckInfosStatus() {
 231
 232                 return lastBeHealthCheckInfos;
 233
 234         }
 235
 236         private List<HealthCheckInfo> getBeHealthCheckInfos() {
 237
 238                 log.trace("In getBeHealthCheckInfos");
 239
 240                 List<HealthCheckInfo> healthCheckInfos = new ArrayList<HealthCheckInfo>();
 241
 242                 // BE
 243                 getBeHealthCheck(healthCheckInfos);
 244
 245                 // ES
 246                 getEsHealthCheck(healthCheckInfos);
 247
 248                 // Titan
 249                 getTitanHealthCheck(healthCheckInfos);
 250
 251                 // Distribution Engine
 252                 getDistributionEngineCheck(healthCheckInfos);
 253
 254                 return healthCheckInfos;
 255         }
 256
 257         private List<HealthCheckInfo> getBeHealthCheck(List<HealthCheckInfo> healthCheckInfos) {
 258                 String appVersion = ExternalConfiguration.getAppVersion();
 259                 String description = "OK";
 260                 healthCheckInfos.add(new HealthCheckInfo(HealthCheckComponent.BE, HealthCheckStatus.UP, appVersion, description));
 261                 return healthCheckInfos;
 262         }
 263
 264         public List<HealthCheckInfo> getEsHealthCheck(List<HealthCheckInfo> healthCheckInfos) {
 265
 266                 // ES health check and version
 267                 HealthCheckStatus healthCheckStatus;
 268                 String description;
 269
 270                 try {
 271                         healthCheckStatus = esHealthCheckDao.getClusterHealthStatus();
 272                 } catch (Exception e) {
 273                         healthCheckStatus = HealthCheckStatus.DOWN;
 274                         description = "ES cluster error: " + e.getMessage();
 275                         healthCheckInfos.add(new HealthCheckInfo(HealthCheckComponent.ES, healthCheckStatus, null, description));
 276                         return healthCheckInfos;
 277                 }
 278                 if (healthCheckStatus.equals(HealthCheckStatus.DOWN)) {
 279                         description = "ES cluster is down";
 280                 } else {
 281                         description = "OK";
 282                 }
 283                 healthCheckInfos.add(new HealthCheckInfo(HealthCheckComponent.ES, healthCheckStatus, null, description));
 284                 return healthCheckInfos;
 285         }
 286
 287         public List<HealthCheckInfo> getTitanHealthCheck(List<HealthCheckInfo> healthCheckInfos) {
 288                 // Titan health check and version
 289                 String description;
 290                 boolean isTitanUp;
 291
 292                 try {
 293                         isTitanUp = titanGenericDao.isGraphOpen();
 294                 } catch (Exception e) {
 295                         description = "Titan error: " + e.getMessage();
 296                         healthCheckInfos.add(new HealthCheckInfo(HealthCheckComponent.TITAN, HealthCheckStatus.DOWN, null, description));
 297                         return healthCheckInfos;
 298                 }
 299                 if (isTitanUp) {
 300                         description = "OK";
 301                         healthCheckInfos.add(new HealthCheckInfo(HealthCheckComponent.TITAN, HealthCheckStatus.UP, null, description));
 302                 } else {
 303                         description = "Titan graph is down";
 304                         healthCheckInfos.add(new HealthCheckInfo(HealthCheckComponent.TITAN, HealthCheckStatus.DOWN, null, description));
 305                 }
 306                 return healthCheckInfos;
 307         }
 308
 309         private void getDistributionEngineCheck(List<HealthCheckInfo> healthCheckInfos) {
 310
 311                 HealthCheckInfo healthCheckInfo = distributionEngineClusterHealth.getHealthCheckInfo();
 312
 313                 healthCheckInfos.add(healthCheckInfo);
 314
 315         }
 316
 317         @PreDestroy
 318         private void destroy() {
 319
 320                 if (scheduledFuture != null) {
 321                         scheduledFuture.cancel(true);
 322                         scheduledFuture = null;
 323                 }
 324
 325                 if (healthCheckScheduler != null) {
 326                         healthCheckScheduler.shutdown();
 327                 }
 328
 329         }
 330
 331         public class HealthCheckScheduledTask implements Runnable {
 332
 333                 List<UebHealthCheckCall> healthCheckCalls = new ArrayList<>();
 334
 335                 public HealthCheckScheduledTask() {
 336
 337                 }
 338
 339                 @Override
 340                 public void run() {
 341
 342                         healthLogger.trace("Executing BE Health Check Task");
 343
 344                         List<HealthCheckInfo> beHealthCheckInfos = getBeHealthCheckInfos();
 345                         boolean healthStatus = getAggregateBeStatus(beHealthCheckInfos);
 346
 347                         boolean lastHealthStatus = getAggregateBeStatus(lastBeHealthCheckInfos);
 348
 349                         if (lastHealthStatus != healthStatus) {
 350                                 log.trace("BE Health State Changed to {}. Issuing alarm / recovery alarm...", healthStatus);
 351
 352                                 lastBeHealthCheckInfos = beHealthCheckInfos;
 353                                 logAlarm(healthStatus);
 354
 355                         } else {
 356                                 // check if we need to update the status's list in case one of
 357                                 // the statuses was changed
 358                                 if (true == anyStatusChanged(beHealthCheckInfos, lastBeHealthCheckInfos)) {
 359                                         lastBeHealthCheckInfos = beHealthCheckInfos;
 360                                 }
 361
 362                         }
 363
 364                 }
 365
 366         }
 367
 368         private void logAlarm(boolean lastHealthState) {
 369                 if (lastHealthState == true) {
 370                         BeEcompErrorManager.getInstance().logBeHealthCheckRecovery(BE_HEALTH_CHECK_STR);
 371                 } else {
 372                         BeEcompErrorManager.getInstance().logBeHealthCheckError(BE_HEALTH_CHECK_STR);
 373                 }
 374         }
 375
 376         private boolean getAggregateBeStatus(List<HealthCheckInfo> beHealthCheckInfos) {
 377
 378                 boolean status = true;
 379
 380                 for (HealthCheckInfo healthCheckInfo : beHealthCheckInfos) {
 381                         if (healthCheckInfo.getHealthCheckStatus().equals(HealthCheckStatus.DOWN) && healthCheckInfo.getHealthCheckComponent() != HealthCheckComponent.DE) {
 382                                 status = false;
 383                                 break;
 384                         }
 385                 }
 386                 return status;
 387         }
 388
 389         public String getSiteMode() {
 390                 return switchoverDetector.getSiteMode();
 391         }
 392
 393         public boolean anyStatusChanged(List<HealthCheckInfo> beHealthCheckInfos, List<HealthCheckInfo> lastBeHealthCheckInfos) {
 394
 395                 boolean result = false;
 396
 397                 if (beHealthCheckInfos != null && lastBeHealthCheckInfos != null) {
 398
 399                         Map<HealthCheckComponent, HealthCheckStatus> currentValues = beHealthCheckInfos.stream().collect(Collectors.toMap(p -> p.getHealthCheckComponent(), p -> p.getHealthCheckStatus()));
 400                         Map<HealthCheckComponent, HealthCheckStatus> lastValues = lastBeHealthCheckInfos.stream().collect(Collectors.toMap(p -> p.getHealthCheckComponent(), p -> p.getHealthCheckStatus()));
 401
 402                         if (currentValues != null && lastValues != null) {
 403                                 int currentSize = currentValues.size();
 404                                 int lastSize = lastValues.size();
 405
 406                                 if (currentSize != lastSize) {
 407                                         result = true;
 408                                 } else {
 409
 410                                         for (Entry<HealthCheckComponent, HealthCheckStatus> entry : currentValues.entrySet()) {
 411                                                 HealthCheckComponent key = entry.getKey();
 412                                                 HealthCheckStatus value = entry.getValue();
 413
 414                                                 if (false == lastValues.containsKey(key)) {
 415                                                         result = true;
 416                                                         break;
 417                                                 }
 418
 419                                                 HealthCheckStatus lastHealthCheckStatus = lastValues.get(key);
 420
 421                                                 if (value != lastHealthCheckStatus) {
 422                                                         result = true;
 423                                                         break;
 424                                                 }
 425                                         }
 426                                 }
 427                         } else if (currentValues == null && lastValues == null) {
 428                                 result = false;
 429                         } else {
 430                                 result = true;
 431                         }
 432
 433                 } else if (beHealthCheckInfos == null && lastBeHealthCheckInfos == null) {
 434                         result = false;
 435                 } else {
 436                         result = true;
 437                 }
 438
 439                 return result;
 440         }
 441 }