catalog-be/src/main/java/org/openecomp/sdc/be/components/health/HealthCheckBusinessLogic.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.be.components.health;
  22
  23 import com.fasterxml.jackson.core.type.TypeReference;
  24 import com.fasterxml.jackson.databind.ObjectMapper;
  25 import com.google.common.annotations.VisibleForTesting;
  26 import org.apache.commons.lang3.tuple.ImmutablePair;
  27 import org.apache.commons.lang3.tuple.Pair;
  28 import org.openecomp.sdc.be.catalog.impl.DmaapProducerHealth;
  29 import org.openecomp.sdc.be.components.distribution.engine.DistributionEngineClusterHealth;
  30 import org.openecomp.sdc.be.components.distribution.engine.DmaapHealth;
  31 import org.openecomp.sdc.be.components.impl.CADIHealthCheck;
  32 import org.openecomp.sdc.be.components.impl.CassandraHealthCheck;
  33 import org.openecomp.sdc.be.config.BeEcompErrorManager;
  34 import org.openecomp.sdc.be.config.Configuration;
  35 import org.openecomp.sdc.be.config.ConfigurationManager;
  36 import org.openecomp.sdc.be.dao.janusgraph.JanusGraphGenericDao;
  37 import org.openecomp.sdc.be.switchover.detector.SwitchoverDetector;
  38 import org.openecomp.sdc.common.api.HealthCheckInfo;
  39 import org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus;
  40 import org.openecomp.sdc.common.http.client.api.HttpRequest;
  41 import org.openecomp.sdc.common.http.client.api.HttpResponse;
  42 import org.openecomp.sdc.common.http.config.HttpClientConfig;
  43 import org.openecomp.sdc.common.http.config.Timeouts;
  44 import org.openecomp.sdc.common.log.elements.LogFieldsMdcHandler;
  45 import org.openecomp.sdc.common.log.wrappers.Logger;
  46 import org.openecomp.sdc.common.util.HealthCheckUtil;
  47 import org.springframework.beans.factory.annotation.Autowired;
  48 import org.springframework.stereotype.Component;
  49
  50 import javax.annotation.PostConstruct;
  51 import javax.annotation.PreDestroy;
  52 import javax.annotation.Resource;
  53 import java.util.ArrayList;
  54 import java.util.HashMap;
  55 import java.util.List;
  56 import java.util.Map;
  57 import java.util.Map.Entry;
  58 import java.util.concurrent.ScheduledExecutorService;
  59 import java.util.concurrent.ScheduledFuture;
  60 import java.util.concurrent.TimeUnit;
  61 import java.util.stream.Collectors;
  62
  63 import static java.lang.String.format;
  64 import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
  65 import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
  66 import static org.apache.http.HttpStatus.SC_OK;
  67 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_BE;
  68 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_CASSANDRA;
  69 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_DCAE;
  70 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_DMAAP_PRODUCER;
  71 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_ECOMP_PORTAL;
  72 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_JANUSGRAPH;
  73 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_ON_BOARDING;
  74 import static org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus.DOWN;
  75 import static org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus.UP;
  76 import static org.openecomp.sdc.common.impl.ExternalConfiguration.getAppVersion;
  77
  78
  79 @Component("healthCheckBusinessLogic")
  80 public class HealthCheckBusinessLogic {
  81
  82     private static String hcUrl = "%s://%s:%s%s";
  83     private static final String BE_HEALTH_CHECK_STR = "beHealthCheck";
  84     private static final String LOG_PARTNER_NAME = "SDC.BE";
  85     private static final String COMPONENT_CHANGED_MESSAGE = "BE Component %s state changed from %s to %s";
  86     private static final Logger log = Logger.getLogger(HealthCheckBusinessLogic.class.getName());
  87     private static final HealthCheckUtil healthCheckUtil = new HealthCheckUtil();
  88     private final ScheduledExecutorService healthCheckScheduler = newSingleThreadScheduledExecutor((Runnable r) -> new Thread(r, "BE-Health-Check-Task"));
  89     private HealthCheckScheduledTask healthCheckScheduledTask = null;
  90     private static LogFieldsMdcHandler mdcFieldsHandler = new LogFieldsMdcHandler();
  91
  92     @Resource
  93     private JanusGraphGenericDao janusGraphGenericDao;
  94     @Resource
  95     private DistributionEngineClusterHealth distributionEngineClusterHealth;
  96     @Resource
  97     private DmaapHealth dmaapHealth;
  98     @Resource
  99     private DmaapProducerHealth dmaapProducerHealth;
 100     @Resource
 101     private CassandraHealthCheck cassandraHealthCheck;
 102     @Resource
 103     private PortalHealthCheckBuilder portalHealthCheck;
 104
 105     @Autowired
 106     private SwitchoverDetector switchoverDetector;
 107     private volatile List<HealthCheckInfo> prevBeHealthCheckInfos = null;
 108     private ScheduledFuture<?> scheduledFuture = null;
 109
 110     @PostConstruct
 111     public void init() {
 112
 113         prevBeHealthCheckInfos = getBeHealthCheckInfos();
 114
 115         log.debug("After initializing prevBeHealthCheckInfos: {}", prevBeHealthCheckInfos);
 116
 117         healthCheckScheduledTask = new HealthCheckScheduledTask();
 118
 119         if (this.scheduledFuture == null) {
 120             this.scheduledFuture = this.healthCheckScheduler.scheduleAtFixedRate(healthCheckScheduledTask, 0, 3, TimeUnit.SECONDS);
 121         }
 122
 123     }
 124
 125     public boolean isDistributionEngineUp() {
 126
 127         HealthCheckInfo healthCheckInfo = distributionEngineClusterHealth.getHealthCheckInfo();
 128         return healthCheckInfo.getHealthCheckStatus() != DOWN;
 129     }
 130
 131     public Pair<Boolean, List<HealthCheckInfo>> getBeHealthCheckInfosStatus() {
 132         Configuration config = ConfigurationManager.getConfigurationManager().getConfiguration();
 133         return new ImmutablePair<>(healthCheckUtil.getAggregateStatus(prevBeHealthCheckInfos, config.getHealthStatusExclude()), prevBeHealthCheckInfos);
 134     }
 135
 136     private List<HealthCheckInfo> getBeHealthCheckInfos() {
 137
 138         log.trace("In getBeHealthCheckInfos");
 139
 140         List<HealthCheckInfo> healthCheckInfos = new ArrayList<>();
 141
 142         //Dmaap
 143         HealthCheckInfo info;
 144         if ((info = getDmaapHealthCheck()) != null) {
 145             healthCheckInfos.add(info);
 146         }
 147
 148         //DmaapProducer
 149         healthCheckInfos.add(getDmaapProducerHealthCheck());
 150
 151         // BE
 152         healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_BE, UP, getAppVersion(), "OK"));
 153
 154         // JanusGraph
 155         healthCheckInfos.add(getJanusGraphHealthCheck());
 156
 157         // Distribution Engine
 158         healthCheckInfos.add(distributionEngineClusterHealth.getHealthCheckInfo());
 159
 160         //Cassandra
 161         healthCheckInfos.add(getCassandraHealthCheck());
 162
 163         // Amdocs
 164         healthCheckInfos.add(getHostedComponentsBeHealthCheck(HC_COMPONENT_ON_BOARDING, buildOnBoardingHealthCheckUrl()));
 165
 166         //DCAE
 167         healthCheckInfos.add(getHostedComponentsBeHealthCheck(HC_COMPONENT_DCAE, buildDcaeHealthCheckUrl()));
 168
 169         //ECOMP Portal
 170         healthCheckInfos.add(portalHealthCheck.getHealthCheckInfo());
 171
 172         //CADI
 173         healthCheckInfos.add(CADIHealthCheck.getCADIHealthCheckInstance().getCADIStatus());
 174
 175         return healthCheckInfos;
 176     }
 177
 178     private HealthCheckInfo getDmaapHealthCheck() {
 179         HealthCheckInfo healthCheckInfo = null;
 180         if(ConfigurationManager.getConfigurationManager().getConfiguration().getDmaapConsumerConfiguration().isActive()){
 181             String appVersion = getAppVersion();
 182             dmaapHealth.getHealthCheckInfo().setVersion(appVersion);
 183             healthCheckInfo = dmaapHealth.getHealthCheckInfo();
 184         } else {
 185           log.debug("Dmaap health check disabled");
 186         }
 187         return healthCheckInfo;
 188     }
 189
 190     private HealthCheckInfo getDmaapProducerHealthCheck() {
 191         if (ConfigurationManager.getConfigurationManager().getConfiguration().getDmaapConsumerConfiguration().isActive()) {
 192             String appVersion = getAppVersion();
 193             dmaapProducerHealth.getHealthCheckInfo().setVersion(appVersion);
 194             return dmaapProducerHealth.getHealthCheckInfo();
 195         } else {
 196             log.debug("Dmaap health check disabled");
 197             String description = ("Dmaap health check disabled");
 198             return new HealthCheckInfo(HC_COMPONENT_DMAAP_PRODUCER, DOWN, null, description);
 199         }
 200     }
 201
 202     public HealthCheckInfo getJanusGraphHealthCheck() {
 203         // JanusGraph health check and version
 204         String description;
 205         boolean isJanusGraphUp;
 206         HealthCheckInfo healthCheckInfo = new HealthCheckInfo(HC_COMPONENT_JANUSGRAPH, DOWN, null, null);
 207
 208         try {
 209             isJanusGraphUp = janusGraphGenericDao.isGraphOpen();
 210         } catch (Exception e) {
 211             description = "JanusGraph error: " + e.getMessage();
 212             healthCheckInfo.setDescription(description);
 213             log.error(description);
 214             return healthCheckInfo;
 215         }
 216         if (isJanusGraphUp) {
 217             description = "OK";
 218             healthCheckInfo.setDescription(description);
 219             healthCheckInfo.setHealthCheckStatus(HealthCheckInfo.HealthCheckStatus.UP);
 220
 221         } else {
 222             description = "JanusGraph graph is down";
 223             healthCheckInfo.setDescription(description);
 224         }
 225         return healthCheckInfo;
 226     }
 227
 228     private HealthCheckInfo getCassandraHealthCheck() {
 229
 230         String description;
 231         boolean isCassandraUp = false;
 232         HealthCheckInfo healthCheckInfo = new HealthCheckInfo(HC_COMPONENT_CASSANDRA, DOWN, null, null);
 233
 234         try {
 235             isCassandraUp = cassandraHealthCheck.getCassandraStatus();
 236         } catch (Exception e) {
 237             description = "Cassandra error: " + e.getMessage();
 238             log.error(description, e);
 239         }
 240         if (isCassandraUp) {
 241             description = "OK";
 242 //            healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_CASSANDRA, UP, null, description));
 243             healthCheckInfo.setHealthCheckStatus(HealthCheckStatus.UP);
 244             healthCheckInfo.setDescription(description);
 245         } else {
 246             description = "Cassandra is down";
 247 //            healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_CASSANDRA, DOWN, null, description));
 248             healthCheckInfo.setDescription(description);
 249         }
 250         return healthCheckInfo;
 251     }
 252
 253     private HealthCheckInfo getHostedComponentsBeHealthCheck(String componentName, String healthCheckUrl) {
 254         HealthCheckStatus healthCheckStatus;
 255         String description;
 256         String version = null;
 257         List<HealthCheckInfo> componentsInfo = new ArrayList<>();
 258         final int timeout = 3000;
 259
 260         if (healthCheckUrl != null) {
 261             try {
 262                 HttpResponse<String> httpResponse = HttpRequest.get(healthCheckUrl, new HttpClientConfig(new Timeouts(timeout, timeout)));
 263                 int statusCode = httpResponse.getStatusCode();
 264                 String aggDescription = "";
 265
 266                 if ((statusCode == SC_OK || statusCode == SC_INTERNAL_SERVER_ERROR) && !componentName.equals(HC_COMPONENT_ECOMP_PORTAL)) {
 267                     String response = httpResponse.getResponse();
 268                     log.trace("{} Health Check response: {}", componentName, response);
 269                     ObjectMapper mapper = new ObjectMapper();
 270                     Map<String, Object> healthCheckMap = mapper.readValue(response, new TypeReference<Map<String, Object>>() {
 271                     });
 272                     version = getVersion(healthCheckMap);
 273                     if (healthCheckMap.containsKey("componentsInfo")) {
 274                         componentsInfo = mapper.convertValue(healthCheckMap.get("componentsInfo"), new TypeReference<List<HealthCheckInfo>>() {
 275                         });
 276                     }
 277                     aggDescription = getAggDescription(componentsInfo, aggDescription);
 278                 } else {
 279                     log.trace("{} Health Check Response code: {}", componentName, statusCode);
 280                 }
 281
 282                 if (statusCode != SC_OK) {
 283                     healthCheckStatus = DOWN;
 284                     description = getDescription(componentName, aggDescription);
 285                     setDescriptionToObject(description, componentsInfo);
 286                 } else {
 287                     healthCheckStatus = UP;
 288                     description = "OK";
 289                 }
 290
 291             } catch (Exception e) {
 292                 log.error("{} unexpected response: ", componentName, e);
 293                 healthCheckStatus = DOWN;
 294                 description = componentName + " unexpected response: " + e.getMessage();
 295                 addToHealthCheckInfoObject(description, componentsInfo);
 296             }
 297         } else {
 298             healthCheckStatus = DOWN;
 299             description = componentName + " health check Configuration is missing";
 300             componentsInfo.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, description));
 301         }
 302         return new HealthCheckInfo(componentName, healthCheckStatus, version, description, componentsInfo);
 303     }
 304
 305     private void addToHealthCheckInfoObject(String description, List<HealthCheckInfo> componentsInfo) {
 306         if (componentsInfo != null && componentsInfo.isEmpty()) {
 307             componentsInfo.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, description));
 308         }
 309     }
 310
 311     private void setDescriptionToObject(String description, List<HealthCheckInfo> componentsInfo) {
 312         if (componentsInfo.isEmpty()) {
 313             componentsInfo.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, description));
 314         }
 315     }
 316
 317     private String getDescription(String componentName, String aggDescription) {
 318         String description;
 319         description = aggDescription.length() > 0
 320                 ? aggDescription
 321                 : componentName + " is Down, specific reason unknown";//No inner component returned DOWN, but the status of HC is still DOWN.
 322         return description;
 323     }
 324
 325     private String getVersion(Map<String, Object> healthCheckMap) {
 326         return healthCheckMap.get("sdcVersion") != null ? healthCheckMap.get("sdcVersion").toString() : null;
 327     }
 328
 329     private String getAggDescription(List<HealthCheckInfo> componentsInfo, String aggDescription) {
 330         if (!componentsInfo.isEmpty()) {
 331             aggDescription = healthCheckUtil.getAggregateDescription(componentsInfo);
 332         } else {
 333             componentsInfo.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, null));
 334         }
 335         return aggDescription;
 336     }
 337
 338     @PreDestroy
 339     protected void destroy() {
 340
 341         if (scheduledFuture != null) {
 342             scheduledFuture.cancel(true);
 343             scheduledFuture = null;
 344         }
 345
 346         if (healthCheckScheduler != null) {
 347             healthCheckScheduler.shutdown();
 348         }
 349
 350     }
 351
 352     private void logAlarm(String componentChangedMsg) {
 353         BeEcompErrorManager.getInstance().logBeHealthCheckRecovery(componentChangedMsg);
 354     }
 355
 356     public String getSiteMode() {
 357         return switchoverDetector.getSiteMode();
 358     }
 359
 360     public boolean anyStatusChanged(List<HealthCheckInfo> beHealthCheckInfos, List<HealthCheckInfo> prevBeHealthCheckInfos) {
 361
 362         boolean result = false;
 363
 364         if (beHealthCheckInfos != null && prevBeHealthCheckInfos != null) {
 365
 366             Map<String, HealthCheckStatus> currentValues = beHealthCheckInfos.stream().collect(Collectors.toMap(HealthCheckInfo::getHealthCheckComponent, HealthCheckInfo::getHealthCheckStatus));
 367             Map<String, HealthCheckStatus> prevValues = prevBeHealthCheckInfos.stream().collect(Collectors.toMap(HealthCheckInfo::getHealthCheckComponent, HealthCheckInfo::getHealthCheckStatus));
 368
 369             if (currentValues != null && prevValues != null) {
 370                 int currentSize = currentValues.size();
 371                 int prevSize = prevValues.size();
 372
 373                 if (currentSize != prevSize) {
 374                     result = true; //extra/missing component
 375                     updateHealthCheckStatusMap(currentValues, prevValues);
 376                 } else {
 377                     result = isHealthStatusChanged(result, currentValues, prevValues);
 378                 }
 379             }
 380
 381         } else if (beHealthCheckInfos == null && prevBeHealthCheckInfos == null) {
 382             result = false;
 383         } else {
 384             writeLogAlarm(prevBeHealthCheckInfos);
 385             result = true;
 386         }
 387
 388         return result;
 389     }
 390
 391     private void writeLogAlarm(List<HealthCheckInfo> prevBeHealthCheckInfos) {
 392         logAlarm(format(COMPONENT_CHANGED_MESSAGE, "", prevBeHealthCheckInfos == null ? "null" : "true", prevBeHealthCheckInfos == null ? "true" : "null"));
 393     }
 394
 395     private boolean isHealthStatusChanged(boolean result, Map<String, HealthCheckStatus> currentValues, Map<String, HealthCheckStatus> prevValues) {
 396         for (Entry<String, HealthCheckStatus> entry : currentValues.entrySet()) {
 397             String key = entry.getKey();
 398             HealthCheckStatus value = entry.getValue();
 399
 400             if (!prevValues.containsKey(key)) {
 401                 result = true; //component missing
 402                 logAlarm(format(COMPONENT_CHANGED_MESSAGE, key, prevValues.get(key), currentValues.get(key)));
 403                 break;
 404             }
 405
 406             HealthCheckStatus prevHealthCheckStatus = prevValues.get(key);
 407
 408             if (value != prevHealthCheckStatus) {
 409                 result = true; //component status changed
 410                 logAlarm(format(COMPONENT_CHANGED_MESSAGE, key, prevValues.get(key), currentValues.get(key)));
 411                 break;
 412             }
 413         }
 414         return result;
 415     }
 416
 417     private void updateHealthCheckStatusMap(Map<String, HealthCheckStatus> currentValues, Map<String, HealthCheckStatus> prevValues) {
 418         Map<String, HealthCheckStatus> notPresent;
 419         if (currentValues.keySet().containsAll(prevValues.keySet())) {
 420             notPresent = new HashMap<>(currentValues);
 421             notPresent.keySet().removeAll(prevValues.keySet());
 422         } else {
 423             notPresent = new HashMap<>(prevValues);
 424             notPresent.keySet().removeAll(currentValues.keySet());
 425         }
 426
 427         for (String component : notPresent.keySet()) {
 428             logAlarm(format(COMPONENT_CHANGED_MESSAGE, component, prevValues.get(component), currentValues.get(component)));
 429         }
 430     }
 431     @VisibleForTesting
 432     String buildOnBoardingHealthCheckUrl() {
 433
 434         Configuration.OnboardingConfig onboardingConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getOnboarding();
 435         if (onboardingConfig != null) {
 436             return String.format(hcUrl, onboardingConfig.getProtocol(), onboardingConfig.getHost(),
 437                     onboardingConfig.getPort(),onboardingConfig.getHealthCheckUri());
 438         }
 439         log.error("Onboarding health check configuration is missing.");
 440         return null;
 441     }
 442
 443     @VisibleForTesting
 444     String buildDcaeHealthCheckUrl() {
 445
 446         Configuration.DcaeConfig dcaeConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getDcae();
 447
 448         if (dcaeConfig != null) {
 449             return String.format(hcUrl, dcaeConfig.getProtocol(), dcaeConfig.getHost(),
 450                     dcaeConfig.getPort(),dcaeConfig.getHealthCheckUri());
 451         }
 452
 453         log.error("DCAE health check configuration is missing.");
 454         return null;
 455     }
 456
 457     public class HealthCheckScheduledTask implements Runnable {
 458         @Override
 459         public void run() {
 460             mdcFieldsHandler.addInfoForErrorAndDebugLogging(LOG_PARTNER_NAME);
 461             Configuration config = ConfigurationManager.getConfigurationManager().getConfiguration();
 462             log.trace("Executing BE Health Check Task");
 463
 464             List<HealthCheckInfo> currentBeHealthCheckInfos = getBeHealthCheckInfos();
 465             boolean healthStatus = healthCheckUtil.getAggregateStatus(currentBeHealthCheckInfos, config.getHealthStatusExclude());
 466
 467             boolean prevHealthStatus = healthCheckUtil.getAggregateStatus(prevBeHealthCheckInfos, config.getHealthStatusExclude());
 468
 469             boolean anyStatusChanged = anyStatusChanged(currentBeHealthCheckInfos, prevBeHealthCheckInfos);
 470
 471             if (prevHealthStatus != healthStatus || anyStatusChanged) {
 472                 log.trace("BE Health State Changed to {}. Issuing alarm / recovery alarm...", healthStatus);
 473
 474                 prevBeHealthCheckInfos = currentBeHealthCheckInfos;
 475                 logAlarm(healthStatus);
 476             }
 477         }
 478
 479         private void logAlarm(boolean prevHealthState) {
 480             if (prevHealthState) {
 481                 BeEcompErrorManager.getInstance().logBeHealthCheckRecovery(BE_HEALTH_CHECK_STR);
 482             } else {
 483                 BeEcompErrorManager.getInstance().logBeHealthCheckError(BE_HEALTH_CHECK_STR);
 484             }
 485         }
 486
 487
 488     }
 489
 490 }