catalog-be/src/main/java/org/openecomp/sdc/be/components/health/HealthCheckBusinessLogic.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.be.components.health;
  22
  23 import com.fasterxml.jackson.core.type.TypeReference;
  24 import com.fasterxml.jackson.databind.ObjectMapper;
  25 import com.google.common.annotations.VisibleForTesting;
  26 import org.apache.commons.lang3.tuple.ImmutablePair;
  27 import org.apache.commons.lang3.tuple.Pair;
  28 import org.openecomp.sdc.be.catalog.impl.DmaapProducerHealth;
  29 import org.openecomp.sdc.be.components.distribution.engine.DistributionEngineClusterHealth;
  30 import org.openecomp.sdc.be.components.distribution.engine.DmaapHealth;
  31 import org.openecomp.sdc.be.components.impl.CADIHealthCheck;
  32 import org.openecomp.sdc.be.components.impl.CassandraHealthCheck;
  33 import org.openecomp.sdc.be.config.BeEcompErrorManager;
  34 import org.openecomp.sdc.be.config.Configuration;
  35 import org.openecomp.sdc.be.config.ConfigurationManager;
  36 import org.openecomp.sdc.be.dao.janusgraph.JanusGraphGenericDao;
  37 import org.openecomp.sdc.be.switchover.detector.SwitchoverDetector;
  38 import org.openecomp.sdc.common.api.HealthCheckInfo;
  39 import org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus;
  40 import org.openecomp.sdc.common.http.client.api.HttpRequest;
  41 import org.openecomp.sdc.common.http.client.api.HttpResponse;
  42 import org.openecomp.sdc.common.http.config.HttpClientConfig;
  43 import org.openecomp.sdc.common.http.config.Timeouts;
  44 import org.openecomp.sdc.common.log.elements.LogFieldsMdcHandler;
  45 import org.openecomp.sdc.common.log.wrappers.Logger;
  46 import org.openecomp.sdc.common.util.HealthCheckUtil;
  47 import org.springframework.beans.factory.annotation.Autowired;
  48 import org.springframework.stereotype.Component;
  49
  50 import javax.annotation.PostConstruct;
  51 import javax.annotation.PreDestroy;
  52 import javax.annotation.Resource;
  53 import java.util.ArrayList;
  54 import java.util.HashMap;
  55 import java.util.List;
  56 import java.util.Map;
  57 import java.util.Map.Entry;
  58 import java.util.concurrent.ScheduledExecutorService;
  59 import java.util.concurrent.ScheduledFuture;
  60 import java.util.concurrent.TimeUnit;
  61 import java.util.stream.Collectors;
  62
  63 import static java.lang.String.format;
  64 import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
  65 import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
  66 import static org.apache.http.HttpStatus.SC_OK;
  67 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_BE;
  68 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_CASSANDRA;
  69 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_DCAE;
  70 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_DMAAP_PRODUCER;
  71 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_ECOMP_PORTAL;
  72 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_JANUSGRAPH;
  73 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_ON_BOARDING;
  74 import static org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus.DOWN;
  75 import static org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus.UP;
  76 import static org.openecomp.sdc.common.impl.ExternalConfiguration.getAppVersion;
  77
  78
  79 @Component("healthCheckBusinessLogic")
  80 public class HealthCheckBusinessLogic {
  81
  82     private static String hcUrl = "%s://%s:%s%s";
  83     private static final String BE_HEALTH_CHECK_STR = "beHealthCheck";
  84     private static final String LOG_PARTNER_NAME = "SDC.BE";
  85     private static final String COMPONENT_CHANGED_MESSAGE = "BE Component %s state changed from %s to %s";
  86     private static final Logger log = Logger.getLogger(HealthCheckBusinessLogic.class.getName());
  87     private static final HealthCheckUtil healthCheckUtil = new HealthCheckUtil();
  88     private final ScheduledExecutorService healthCheckScheduler = newSingleThreadScheduledExecutor((Runnable r) -> new Thread(r, "BE-Health-Check-Task"));
  89     private HealthCheckScheduledTask healthCheckScheduledTask = null;
  90     private static LogFieldsMdcHandler mdcFieldsHandler = new LogFieldsMdcHandler();
  91
  92     @Resource
  93     private JanusGraphGenericDao janusGraphGenericDao;
  94     @Resource
  95     private DistributionEngineClusterHealth distributionEngineClusterHealth;
  96     @Resource
  97     private DmaapHealth dmaapHealth;
  98     @Resource
  99     private DmaapProducerHealth dmaapProducerHealth;
 100     @Resource
 101     private CassandraHealthCheck cassandraHealthCheck;
 102     @Resource
 103     private PortalHealthCheckBuilder portalHealthCheck;
 104
 105     @Autowired
 106     private SwitchoverDetector switchoverDetector;
 107     private volatile List<HealthCheckInfo> prevBeHealthCheckInfos = null;
 108     private ScheduledFuture<?> scheduledFuture = null;
 109
 110     @PostConstruct
 111     public void init() {
 112
 113         prevBeHealthCheckInfos = getBeHealthCheckInfos();
 114
 115         log.debug("After initializing prevBeHealthCheckInfos: {}", prevBeHealthCheckInfos);
 116
 117         healthCheckScheduledTask = new HealthCheckScheduledTask();
 118
 119         if (this.scheduledFuture == null) {
 120             this.scheduledFuture = this.healthCheckScheduler.scheduleAtFixedRate(healthCheckScheduledTask, 0, 3, TimeUnit.SECONDS);
 121         }
 122
 123     }
 124
 125     public boolean isDistributionEngineUp() {
 126
 127         HealthCheckInfo healthCheckInfo = distributionEngineClusterHealth.getHealthCheckInfo();
 128         return healthCheckInfo.getHealthCheckStatus() != DOWN;
 129     }
 130
 131     public Pair<Boolean, List<HealthCheckInfo>> getBeHealthCheckInfosStatus() {
 132         Configuration config = ConfigurationManager.getConfigurationManager().getConfiguration();
 133         return new ImmutablePair<>(healthCheckUtil.getAggregateStatus(prevBeHealthCheckInfos, config.getHealthStatusExclude()), prevBeHealthCheckInfos);
 134     }
 135
 136     private List<HealthCheckInfo> getBeHealthCheckInfos() {
 137
 138         log.trace("In getBeHealthCheckInfos");
 139
 140         List<HealthCheckInfo> healthCheckInfos = new ArrayList<>();
 141
 142         //Dmaap
 143         HealthCheckInfo info;
 144         if ((info = getDmaapHealthCheck()) != null) {
 145             healthCheckInfos.add(info);
 146         }
 147
 148         //DmaapProducer
 149         healthCheckInfos.add(getDmaapProducerHealthCheck());
 150
 151         // BE
 152         healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_BE, UP, getAppVersion(), "OK"));
 153
 154         // JanusGraph
 155         healthCheckInfos.add(getJanusGraphHealthCheck());
 156
 157         // Distribution Engine
 158         healthCheckInfos.add(distributionEngineClusterHealth.getHealthCheckInfo());
 159
 160         //Cassandra
 161         healthCheckInfos.add(getCassandraHealthCheck());
 162
 163         // Amdocs
 164         healthCheckInfos.add(getHostedComponentsBeHealthCheck(HC_COMPONENT_ON_BOARDING, buildOnBoardingHealthCheckUrl()));
 165
 166         //DCAE
 167         healthCheckInfos.add(getHostedComponentsBeHealthCheck(HC_COMPONENT_DCAE, buildDcaeHealthCheckUrl()));
 168
 169         //ECOMP Portal
 170         healthCheckInfos.add(portalHealthCheck.getHealthCheckInfo());
 171
 172         //CADI
 173         healthCheckInfos.add(CADIHealthCheck.getCADIHealthCheckInstance().getCADIStatus());
 174
 175         return healthCheckInfos;
 176     }
 177
 178     private HealthCheckInfo getDmaapHealthCheck() {
 179         HealthCheckInfo healthCheckInfo = null;
 180         if(ConfigurationManager.getConfigurationManager().getConfiguration().getDmaapConsumerConfiguration().isActive()){
 181             String appVersion = getAppVersion();
 182             dmaapHealth.getHealthCheckInfo().setVersion(appVersion);
 183             healthCheckInfo = dmaapHealth.getHealthCheckInfo();
 184         } else {
 185           log.debug("Dmaap health check disabled");
 186         }
 187         return healthCheckInfo;
 188     }
 189
 190     private HealthCheckInfo getDmaapProducerHealthCheck() {
 191         HealthCheckInfo healthCheckInfo = null;
 192         if(ConfigurationManager.getConfigurationManager().getConfiguration().getDmaapConsumerConfiguration().isActive()){
 193             String appVersion = getAppVersion();
 194         dmaapProducerHealth.getHealthCheckInfo().setVersion(appVersion);
 195         healthCheckInfo = dmaapProducerHealth.getHealthCheckInfo();
 196         } else {
 197           log.debug("Dmaap health check disabled");
 198             String description = ("Dmaap health check disabled");
 199             healthCheckInfo = new HealthCheckInfo(HC_COMPONENT_DMAAP_PRODUCER, DOWN, null, description);
 200         }
 201         return healthCheckInfo;
 202     }
 203
 204     public HealthCheckInfo getJanusGraphHealthCheck() {
 205         // JanusGraph health check and version
 206         String description;
 207         boolean isJanusGraphUp;
 208         HealthCheckInfo healthCheckInfo = new HealthCheckInfo(HC_COMPONENT_JANUSGRAPH, DOWN, null, null);
 209
 210         try {
 211             isJanusGraphUp = janusGraphGenericDao.isGraphOpen();
 212         } catch (Exception e) {
 213             description = "JanusGraph error: " + e.getMessage();
 214             healthCheckInfo.setDescription(description);
 215             log.error(description);
 216             return healthCheckInfo;
 217         }
 218         if (isJanusGraphUp) {
 219             description = "OK";
 220             healthCheckInfo.setDescription(description);
 221             healthCheckInfo.setHealthCheckStatus(HealthCheckInfo.HealthCheckStatus.UP);
 222
 223         } else {
 224             description = "JanusGraph graph is down";
 225             healthCheckInfo.setDescription(description);
 226         }
 227         return healthCheckInfo;
 228     }
 229
 230     private HealthCheckInfo getCassandraHealthCheck() {
 231
 232         String description;
 233         boolean isCassandraUp = false;
 234         HealthCheckInfo healthCheckInfo = new HealthCheckInfo(HC_COMPONENT_CASSANDRA, DOWN, null, null);
 235
 236         try {
 237             isCassandraUp = cassandraHealthCheck.getCassandraStatus();
 238         } catch (Exception e) {
 239             description = "Cassandra error: " + e.getMessage();
 240             log.error(description, e);
 241         }
 242         if (isCassandraUp) {
 243             description = "OK";
 244 //            healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_CASSANDRA, UP, null, description));
 245             healthCheckInfo.setHealthCheckStatus(HealthCheckStatus.UP);
 246             healthCheckInfo.setDescription(description);
 247         } else {
 248             description = "Cassandra is down";
 249 //            healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_CASSANDRA, DOWN, null, description));
 250             healthCheckInfo.setDescription(description);
 251         }
 252         return healthCheckInfo;
 253     }
 254
 255     private HealthCheckInfo getHostedComponentsBeHealthCheck(String componentName, String healthCheckUrl) {
 256         HealthCheckStatus healthCheckStatus;
 257         String description;
 258         String version = null;
 259         List<HealthCheckInfo> componentsInfo = new ArrayList<>();
 260         final int timeout = 3000;
 261
 262         if (healthCheckUrl != null) {
 263             try {
 264                 HttpResponse<String> httpResponse = HttpRequest.get(healthCheckUrl, new HttpClientConfig(new Timeouts(timeout, timeout)));
 265                 int statusCode = httpResponse.getStatusCode();
 266                 String aggDescription = "";
 267
 268                 if ((statusCode == SC_OK || statusCode == SC_INTERNAL_SERVER_ERROR) && !componentName.equals(HC_COMPONENT_ECOMP_PORTAL)) {
 269                     String response = httpResponse.getResponse();
 270                     log.trace("{} Health Check response: {}", componentName, response);
 271                     ObjectMapper mapper = new ObjectMapper();
 272                     Map<String, Object> healthCheckMap = mapper.readValue(response, new TypeReference<Map<String, Object>>() {
 273                     });
 274                     version = getVersion(healthCheckMap);
 275                     if (healthCheckMap.containsKey("componentsInfo")) {
 276                         componentsInfo = mapper.convertValue(healthCheckMap.get("componentsInfo"), new TypeReference<List<HealthCheckInfo>>() {
 277                         });
 278                     }
 279                     aggDescription = getAggDescription(componentsInfo, aggDescription);
 280                 } else {
 281                     log.trace("{} Health Check Response code: {}", componentName, statusCode);
 282                 }
 283
 284                 if (statusCode != SC_OK) {
 285                     healthCheckStatus = DOWN;
 286                     description = getDescription(componentName, aggDescription);
 287                     setDescriptionToObject(description, componentsInfo);
 288                 } else {
 289                     healthCheckStatus = UP;
 290                     description = "OK";
 291                 }
 292
 293             } catch (Exception e) {
 294                 log.error("{} unexpected response: ", componentName, e);
 295                 healthCheckStatus = DOWN;
 296                 description = componentName + " unexpected response: " + e.getMessage();
 297                 addToHealthCheckInfoObject(description, componentsInfo);
 298             }
 299         } else {
 300             healthCheckStatus = DOWN;
 301             description = componentName + " health check Configuration is missing";
 302             componentsInfo.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, description));
 303         }
 304         return new HealthCheckInfo(componentName, healthCheckStatus, version, description, componentsInfo);
 305     }
 306
 307     private void addToHealthCheckInfoObject(String description, List<HealthCheckInfo> componentsInfo) {
 308         if (componentsInfo != null && componentsInfo.isEmpty()) {
 309             componentsInfo.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, description));
 310         }
 311     }
 312
 313     private void setDescriptionToObject(String description, List<HealthCheckInfo> componentsInfo) {
 314         if (componentsInfo.isEmpty()) {
 315             componentsInfo.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, description));
 316         }
 317     }
 318
 319     private String getDescription(String componentName, String aggDescription) {
 320         String description;
 321         description = aggDescription.length() > 0
 322                 ? aggDescription
 323                 : componentName + " is Down, specific reason unknown";//No inner component returned DOWN, but the status of HC is still DOWN.
 324         return description;
 325     }
 326
 327     private String getVersion(Map<String, Object> healthCheckMap) {
 328         return healthCheckMap.get("sdcVersion") != null ? healthCheckMap.get("sdcVersion").toString() : null;
 329     }
 330
 331     private String getAggDescription(List<HealthCheckInfo> componentsInfo, String aggDescription) {
 332         if (!componentsInfo.isEmpty()) {
 333             aggDescription = healthCheckUtil.getAggregateDescription(componentsInfo);
 334         } else {
 335             componentsInfo.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, null));
 336         }
 337         return aggDescription;
 338     }
 339
 340     @PreDestroy
 341     protected void destroy() {
 342
 343         if (scheduledFuture != null) {
 344             scheduledFuture.cancel(true);
 345             scheduledFuture = null;
 346         }
 347
 348         if (healthCheckScheduler != null) {
 349             healthCheckScheduler.shutdown();
 350         }
 351
 352     }
 353
 354     private void logAlarm(String componentChangedMsg) {
 355         BeEcompErrorManager.getInstance().logBeHealthCheckRecovery(componentChangedMsg);
 356     }
 357
 358     public String getSiteMode() {
 359         return switchoverDetector.getSiteMode();
 360     }
 361
 362     public boolean anyStatusChanged(List<HealthCheckInfo> beHealthCheckInfos, List<HealthCheckInfo> prevBeHealthCheckInfos) {
 363
 364         boolean result = false;
 365
 366         if (beHealthCheckInfos != null && prevBeHealthCheckInfos != null) {
 367
 368             Map<String, HealthCheckStatus> currentValues = beHealthCheckInfos.stream().collect(Collectors.toMap(HealthCheckInfo::getHealthCheckComponent, HealthCheckInfo::getHealthCheckStatus));
 369             Map<String, HealthCheckStatus> prevValues = prevBeHealthCheckInfos.stream().collect(Collectors.toMap(HealthCheckInfo::getHealthCheckComponent, HealthCheckInfo::getHealthCheckStatus));
 370
 371             if (currentValues != null && prevValues != null) {
 372                 int currentSize = currentValues.size();
 373                 int prevSize = prevValues.size();
 374
 375                 if (currentSize != prevSize) {
 376                     result = true; //extra/missing component
 377                     updateHealthCheckStatusMap(currentValues, prevValues);
 378                 } else {
 379                     result = isHealthStatusChanged(result, currentValues, prevValues);
 380                 }
 381             }
 382
 383         } else if (beHealthCheckInfos == null && prevBeHealthCheckInfos == null) {
 384             result = false;
 385         } else {
 386             writeLogAlarm(prevBeHealthCheckInfos);
 387             result = true;
 388         }
 389
 390         return result;
 391     }
 392
 393     private void writeLogAlarm(List<HealthCheckInfo> prevBeHealthCheckInfos) {
 394         logAlarm(format(COMPONENT_CHANGED_MESSAGE, "", prevBeHealthCheckInfos == null ? "null" : "true", prevBeHealthCheckInfos == null ? "true" : "null"));
 395     }
 396
 397     private boolean isHealthStatusChanged(boolean result, Map<String, HealthCheckStatus> currentValues, Map<String, HealthCheckStatus> prevValues) {
 398         for (Entry<String, HealthCheckStatus> entry : currentValues.entrySet()) {
 399             String key = entry.getKey();
 400             HealthCheckStatus value = entry.getValue();
 401
 402             if (!prevValues.containsKey(key)) {
 403                 result = true; //component missing
 404                 logAlarm(format(COMPONENT_CHANGED_MESSAGE, key, prevValues.get(key), currentValues.get(key)));
 405                 break;
 406             }
 407
 408             HealthCheckStatus prevHealthCheckStatus = prevValues.get(key);
 409
 410             if (value != prevHealthCheckStatus) {
 411                 result = true; //component status changed
 412                 logAlarm(format(COMPONENT_CHANGED_MESSAGE, key, prevValues.get(key), currentValues.get(key)));
 413                 break;
 414             }
 415         }
 416         return result;
 417     }
 418
 419     private void updateHealthCheckStatusMap(Map<String, HealthCheckStatus> currentValues, Map<String, HealthCheckStatus> prevValues) {
 420         Map<String, HealthCheckStatus> notPresent;
 421         if (currentValues.keySet().containsAll(prevValues.keySet())) {
 422             notPresent = new HashMap<>(currentValues);
 423             notPresent.keySet().removeAll(prevValues.keySet());
 424         } else {
 425             notPresent = new HashMap<>(prevValues);
 426             notPresent.keySet().removeAll(currentValues.keySet());
 427         }
 428
 429         for (String component : notPresent.keySet()) {
 430             logAlarm(format(COMPONENT_CHANGED_MESSAGE, component, prevValues.get(component), currentValues.get(component)));
 431         }
 432     }
 433     @VisibleForTesting
 434     String buildOnBoardingHealthCheckUrl() {
 435
 436         Configuration.OnboardingConfig onboardingConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getOnboarding();
 437         if (onboardingConfig != null) {
 438             return String.format(hcUrl, onboardingConfig.getProtocol(), onboardingConfig.getHost(),
 439                     onboardingConfig.getPort(),onboardingConfig.getHealthCheckUri());
 440         }
 441         log.error("Onboarding health check configuration is missing.");
 442         return null;
 443     }
 444
 445     @VisibleForTesting
 446     String buildDcaeHealthCheckUrl() {
 447
 448         Configuration.DcaeConfig dcaeConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getDcae();
 449
 450         if (dcaeConfig != null) {
 451             return String.format(hcUrl, dcaeConfig.getProtocol(), dcaeConfig.getHost(),
 452                     dcaeConfig.getPort(),dcaeConfig.getHealthCheckUri());
 453         }
 454
 455         log.error("DCAE health check configuration is missing.");
 456         return null;
 457     }
 458
 459     public class HealthCheckScheduledTask implements Runnable {
 460         @Override
 461         public void run() {
 462             mdcFieldsHandler.addInfoForErrorAndDebugLogging(LOG_PARTNER_NAME);
 463             Configuration config = ConfigurationManager.getConfigurationManager().getConfiguration();
 464             log.trace("Executing BE Health Check Task");
 465
 466             List<HealthCheckInfo> currentBeHealthCheckInfos = getBeHealthCheckInfos();
 467             boolean healthStatus = healthCheckUtil.getAggregateStatus(currentBeHealthCheckInfos, config.getHealthStatusExclude());
 468
 469             boolean prevHealthStatus = healthCheckUtil.getAggregateStatus(prevBeHealthCheckInfos, config.getHealthStatusExclude());
 470
 471             boolean anyStatusChanged = anyStatusChanged(currentBeHealthCheckInfos, prevBeHealthCheckInfos);
 472
 473             if (prevHealthStatus != healthStatus || anyStatusChanged) {
 474                 log.trace("BE Health State Changed to {}. Issuing alarm / recovery alarm...", healthStatus);
 475
 476                 prevBeHealthCheckInfos = currentBeHealthCheckInfos;
 477                 logAlarm(healthStatus);
 478             }
 479         }
 480
 481         private void logAlarm(boolean prevHealthState) {
 482             if (prevHealthState) {
 483                 BeEcompErrorManager.getInstance().logBeHealthCheckRecovery(BE_HEALTH_CHECK_STR);
 484             } else {
 485                 BeEcompErrorManager.getInstance().logBeHealthCheckError(BE_HEALTH_CHECK_STR);
 486             }
 487         }
 488
 489
 490     }
 491
 492 }