catalog-be/src/main/java/org/openecomp/sdc/be/components/health/HealthCheckBusinessLogic.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20 package org.openecomp.sdc.be.components.health;
  21
  22 import static java.lang.String.format;
  23 import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
  24 import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
  25 import static org.apache.http.HttpStatus.SC_OK;
  26 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_BE;
  27 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_CASSANDRA;
  28 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_DMAAP_PRODUCER;
  29 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_ECOMP_PORTAL;
  30 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_JANUSGRAPH;
  31 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_ON_BOARDING;
  32 import static org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus.DOWN;
  33 import static org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus.UP;
  34 import static org.openecomp.sdc.common.impl.ExternalConfiguration.getAppVersion;
  35
  36 import com.fasterxml.jackson.core.type.TypeReference;
  37 import com.fasterxml.jackson.databind.ObjectMapper;
  38 import com.google.common.annotations.VisibleForTesting;
  39 import java.util.ArrayList;
  40 import java.util.HashMap;
  41 import java.util.List;
  42 import java.util.Map;
  43 import java.util.Map.Entry;
  44 import java.util.concurrent.ScheduledExecutorService;
  45 import java.util.concurrent.ScheduledFuture;
  46 import java.util.concurrent.TimeUnit;
  47 import java.util.stream.Collectors;
  48 import javax.annotation.PostConstruct;
  49 import javax.annotation.PreDestroy;
  50 import javax.annotation.Resource;
  51 import org.apache.commons.lang3.tuple.ImmutablePair;
  52 import org.apache.commons.lang3.tuple.Pair;
  53 import org.openecomp.sdc.be.catalog.impl.DmaapProducerHealth;
  54 import org.openecomp.sdc.be.components.distribution.engine.DistributionEngineClusterHealth;
  55 import org.openecomp.sdc.be.components.distribution.engine.DmaapHealth;
  56 import org.openecomp.sdc.be.components.impl.CADIHealthCheck;
  57 import org.openecomp.sdc.be.components.impl.CassandraHealthCheck;
  58 import org.openecomp.sdc.be.config.BeEcompErrorManager;
  59 import org.openecomp.sdc.be.config.Configuration;
  60 import org.openecomp.sdc.be.config.ConfigurationManager;
  61 import org.openecomp.sdc.be.dao.janusgraph.JanusGraphGenericDao;
  62 import org.openecomp.sdc.be.switchover.detector.SwitchoverDetector;
  63 import org.openecomp.sdc.common.api.HealthCheckInfo;
  64 import org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus;
  65 import org.openecomp.sdc.common.http.client.api.HttpRequest;
  66 import org.openecomp.sdc.common.http.client.api.HttpResponse;
  67 import org.openecomp.sdc.common.http.config.HttpClientConfig;
  68 import org.openecomp.sdc.common.http.config.Timeouts;
  69 import org.openecomp.sdc.common.log.elements.LogFieldsMdcHandler;
  70 import org.openecomp.sdc.common.log.wrappers.Logger;
  71 import org.openecomp.sdc.common.util.HealthCheckUtil;
  72 import org.springframework.beans.factory.annotation.Autowired;
  73 import org.springframework.stereotype.Component;
  74
  75 @Component("healthCheckBusinessLogic")
  76 public class HealthCheckBusinessLogic {
  77
  78     private static String hcUrl = "%s://%s:%s%s";
  79     private static final String BE_HEALTH_CHECK_STR = "beHealthCheck";
  80     private static final String LOG_PARTNER_NAME = "SDC.BE";
  81     private static final String COMPONENT_CHANGED_MESSAGE = "BE Component %s state changed from %s to %s";
  82     private static final Logger log = Logger.getLogger(HealthCheckBusinessLogic.class.getName());
  83     private static final HealthCheckUtil healthCheckUtil = new HealthCheckUtil();
  84     private final ScheduledExecutorService healthCheckScheduler = newSingleThreadScheduledExecutor(
  85         (Runnable r) -> new Thread(r, "BE-Health-Check-Task"));
  86     private HealthCheckScheduledTask healthCheckScheduledTask = null;
  87     private static LogFieldsMdcHandler mdcFieldsHandler = new LogFieldsMdcHandler();
  88     @Resource
  89     private JanusGraphGenericDao janusGraphGenericDao;
  90     @Resource
  91     private DistributionEngineClusterHealth distributionEngineClusterHealth;
  92     @Resource
  93     private DmaapHealth dmaapHealth;
  94     @Resource
  95     private DmaapProducerHealth dmaapProducerHealth;
  96     @Resource
  97     private CassandraHealthCheck cassandraHealthCheck;
  98     @Resource
  99     private PortalHealthCheckBuilder portalHealthCheck;
 100     @Autowired
 101     private SwitchoverDetector switchoverDetector;
 102     private volatile List<HealthCheckInfo> prevBeHealthCheckInfos = null;
 103     private ScheduledFuture<?> scheduledFuture = null;
 104
 105     @PostConstruct
 106     public void init() {
 107         prevBeHealthCheckInfos = getBeHealthCheckInfos();
 108         log.debug("After initializing prevBeHealthCheckInfos: {}", prevBeHealthCheckInfos);
 109         healthCheckScheduledTask = new HealthCheckScheduledTask();
 110         if (this.scheduledFuture == null) {
 111             this.scheduledFuture = this.healthCheckScheduler.scheduleAtFixedRate(healthCheckScheduledTask, 0, 3, TimeUnit.SECONDS);
 112         }
 113     }
 114
 115     public boolean isDistributionEngineUp() {
 116         HealthCheckInfo healthCheckInfo = distributionEngineClusterHealth.getHealthCheckInfo();
 117         return healthCheckInfo.getHealthCheckStatus() != DOWN;
 118     }
 119
 120     public Pair<Boolean, List<HealthCheckInfo>> getBeHealthCheckInfosStatus() {
 121         Configuration config = ConfigurationManager.getConfigurationManager().getConfiguration();
 122         return new ImmutablePair<>(healthCheckUtil.getAggregateStatus(prevBeHealthCheckInfos, config.getHealthStatusExclude()),
 123             prevBeHealthCheckInfos);
 124     }
 125
 126     private List<HealthCheckInfo> getBeHealthCheckInfos() {
 127         log.trace("In getBeHealthCheckInfos");
 128         List<HealthCheckInfo> healthCheckInfos = new ArrayList<>();
 129         //Dmaap
 130         HealthCheckInfo info;
 131         if ((info = getDmaapHealthCheck()) != null) {
 132             healthCheckInfos.add(info);
 133         }
 134         //DmaapProducer
 135         healthCheckInfos.add(getDmaapProducerHealthCheck());
 136         // BE
 137         healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_BE, UP, getAppVersion(), "OK"));
 138         // JanusGraph
 139         healthCheckInfos.add(getJanusGraphHealthCheck());
 140         // Distribution Engine
 141         healthCheckInfos.add(distributionEngineClusterHealth.getHealthCheckInfo());
 142         //Cassandra
 143         healthCheckInfos.add(getCassandraHealthCheck());
 144         // Amdocs
 145         healthCheckInfos.add(getHostedComponentsBeHealthCheck(HC_COMPONENT_ON_BOARDING, buildOnBoardingHealthCheckUrl()));
 146         //ECOMP Portal
 147         healthCheckInfos.add(portalHealthCheck.getHealthCheckInfo());
 148         //CADI
 149         healthCheckInfos.add(CADIHealthCheck.getCADIHealthCheckInstance().getCADIStatus());
 150         return healthCheckInfos;
 151     }
 152
 153     private HealthCheckInfo getDmaapHealthCheck() {
 154         HealthCheckInfo healthCheckInfo = null;
 155         if (ConfigurationManager.getConfigurationManager().getConfiguration().getDmaapConsumerConfiguration().isActive()) {
 156             String appVersion = getAppVersion();
 157             dmaapHealth.getHealthCheckInfo().setVersion(appVersion);
 158             healthCheckInfo = dmaapHealth.getHealthCheckInfo();
 159         } else {
 160             log.debug("Dmaap health check disabled");
 161         }
 162         return healthCheckInfo;
 163     }
 164
 165     private HealthCheckInfo getDmaapProducerHealthCheck() {
 166         if (ConfigurationManager.getConfigurationManager().getConfiguration().getDmaapConsumerConfiguration().isActive()) {
 167             String appVersion = getAppVersion();
 168             dmaapProducerHealth.getHealthCheckInfo().setVersion(appVersion);
 169             return dmaapProducerHealth.getHealthCheckInfo();
 170         } else {
 171             log.debug("Dmaap health check disabled");
 172             String description = ("Dmaap health check disabled");
 173             return new HealthCheckInfo(HC_COMPONENT_DMAAP_PRODUCER, DOWN, null, description);
 174         }
 175     }
 176
 177     public HealthCheckInfo getJanusGraphHealthCheck() {
 178         // JanusGraph health check and version
 179         String description;
 180         boolean isJanusGraphUp;
 181         HealthCheckInfo healthCheckInfo = new HealthCheckInfo(HC_COMPONENT_JANUSGRAPH, DOWN, null, null);
 182         try {
 183             isJanusGraphUp = janusGraphGenericDao.isGraphOpen();
 184         } catch (Exception e) {
 185             description = "JanusGraph error: " + e.getMessage();
 186             healthCheckInfo.setDescription(description);
 187             log.error(description);
 188             return healthCheckInfo;
 189         }
 190         if (isJanusGraphUp) {
 191             description = "OK";
 192             healthCheckInfo.setDescription(description);
 193             healthCheckInfo.setHealthCheckStatus(HealthCheckInfo.HealthCheckStatus.UP);
 194         } else {
 195             description = "JanusGraph graph is down";
 196             healthCheckInfo.setDescription(description);
 197         }
 198         return healthCheckInfo;
 199     }
 200
 201     private HealthCheckInfo getCassandraHealthCheck() {
 202         String description;
 203         boolean isCassandraUp = false;
 204         HealthCheckInfo healthCheckInfo = new HealthCheckInfo(HC_COMPONENT_CASSANDRA, DOWN, null, null);
 205         try {
 206             isCassandraUp = cassandraHealthCheck.getCassandraStatus();
 207         } catch (Exception e) {
 208             description = "Cassandra error: " + e.getMessage();
 209             log.error(description, e);
 210         }
 211         if (isCassandraUp) {
 212             description = "OK";
 213 //            healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_CASSANDRA, UP, null, description));
 214             healthCheckInfo.setHealthCheckStatus(HealthCheckStatus.UP);
 215             healthCheckInfo.setDescription(description);
 216         } else {
 217             description = "Cassandra is down";
 218 //            healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_CASSANDRA, DOWN, null, description));
 219             healthCheckInfo.setDescription(description);
 220         }
 221         return healthCheckInfo;
 222     }
 223
 224     private HealthCheckInfo getHostedComponentsBeHealthCheck(String componentName, String healthCheckUrl) {
 225         HealthCheckStatus healthCheckStatus;
 226         String description;
 227         String version = null;
 228         List<HealthCheckInfo> componentsInfo = new ArrayList<>();
 229         final int timeout = 3000;
 230         if (healthCheckUrl != null) {
 231             try {
 232                 HttpResponse<String> httpResponse = HttpRequest.get(healthCheckUrl, new HttpClientConfig(new Timeouts(timeout, timeout)));
 233                 int statusCode = httpResponse.getStatusCode();
 234                 String aggDescription = "";
 235                 if ((statusCode == SC_OK || statusCode == SC_INTERNAL_SERVER_ERROR) && !componentName.equals(HC_COMPONENT_ECOMP_PORTAL)) {
 236                     String response = httpResponse.getResponse();
 237                     log.trace("{} Health Check response: {}", componentName, response);
 238                     ObjectMapper mapper = new ObjectMapper();
 239                     Map<String, Object> healthCheckMap = mapper.readValue(response, new TypeReference<Map<String, Object>>() {
 240                     });
 241                     version = getVersion(healthCheckMap);
 242                     if (healthCheckMap.containsKey("componentsInfo")) {
 243                         componentsInfo = mapper.convertValue(healthCheckMap.get("componentsInfo"), new TypeReference<List<HealthCheckInfo>>() {
 244                         });
 245                     }
 246                     aggDescription = getAggDescription(componentsInfo, aggDescription);
 247                 } else {
 248                     log.trace("{} Health Check Response code: {}", componentName, statusCode);
 249                 }
 250                 if (statusCode != SC_OK) {
 251                     healthCheckStatus = DOWN;
 252                     description = getDescription(componentName, aggDescription);
 253                     setDescriptionToObject(description, componentsInfo);
 254                 } else {
 255                     healthCheckStatus = UP;
 256                     description = "OK";
 257                 }
 258             } catch (Exception e) {
 259                 log.error("{} unexpected response: ", componentName, e);
 260                 healthCheckStatus = DOWN;
 261                 description = componentName + " unexpected response: " + e.getMessage();
 262                 addToHealthCheckInfoObject(description, componentsInfo);
 263             }
 264         } else {
 265             healthCheckStatus = DOWN;
 266             description = componentName + " health check Configuration is missing";
 267             componentsInfo.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, description));
 268         }
 269         return new HealthCheckInfo(componentName, healthCheckStatus, version, description, componentsInfo);
 270     }
 271
 272     private void addToHealthCheckInfoObject(String description, List<HealthCheckInfo> componentsInfo) {
 273         if (componentsInfo != null && componentsInfo.isEmpty()) {
 274             componentsInfo.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, description));
 275         }
 276     }
 277
 278     private void setDescriptionToObject(String description, List<HealthCheckInfo> componentsInfo) {
 279         if (componentsInfo.isEmpty()) {
 280             componentsInfo.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, description));
 281         }
 282     }
 283
 284     private String getDescription(String componentName, String aggDescription) {
 285         String description;
 286         description = aggDescription.length() > 0 ? aggDescription
 287             : componentName + " is Down, specific reason unknown";//No inner component returned DOWN, but the status of HC is still DOWN.
 288         return description;
 289     }
 290
 291     private String getVersion(Map<String, Object> healthCheckMap) {
 292         return healthCheckMap.get("sdcVersion") != null ? healthCheckMap.get("sdcVersion").toString() : null;
 293     }
 294
 295     private String getAggDescription(List<HealthCheckInfo> componentsInfo, String aggDescription) {
 296         if (!componentsInfo.isEmpty()) {
 297             aggDescription = healthCheckUtil.getAggregateDescription(componentsInfo);
 298         } else {
 299             componentsInfo.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, null));
 300         }
 301         return aggDescription;
 302     }
 303
 304     @PreDestroy
 305     protected void destroy() {
 306         if (scheduledFuture != null) {
 307             scheduledFuture.cancel(true);
 308             scheduledFuture = null;
 309         }
 310         if (healthCheckScheduler != null) {
 311             healthCheckScheduler.shutdown();
 312         }
 313     }
 314
 315     private void logAlarm(String componentChangedMsg) {
 316         BeEcompErrorManager.getInstance().logBeHealthCheckRecovery(componentChangedMsg);
 317     }
 318
 319     public String getSiteMode() {
 320         return switchoverDetector.getSiteMode();
 321     }
 322
 323     public boolean anyStatusChanged(List<HealthCheckInfo> beHealthCheckInfos, List<HealthCheckInfo> prevBeHealthCheckInfos) {
 324         boolean result = false;
 325         if (beHealthCheckInfos != null && prevBeHealthCheckInfos != null) {
 326             Map<String, HealthCheckStatus> currentValues = beHealthCheckInfos.stream()
 327                 .collect(Collectors.toMap(HealthCheckInfo::getHealthCheckComponent, HealthCheckInfo::getHealthCheckStatus));
 328             Map<String, HealthCheckStatus> prevValues = prevBeHealthCheckInfos.stream()
 329                 .collect(Collectors.toMap(HealthCheckInfo::getHealthCheckComponent, HealthCheckInfo::getHealthCheckStatus));
 330             if (currentValues != null && prevValues != null) {
 331                 int currentSize = currentValues.size();
 332                 int prevSize = prevValues.size();
 333                 if (currentSize != prevSize) {
 334                     result = true; //extra/missing component
 335                     updateHealthCheckStatusMap(currentValues, prevValues);
 336                 } else {
 337                     result = isHealthStatusChanged(result, currentValues, prevValues);
 338                 }
 339             }
 340         } else if (beHealthCheckInfos == null && prevBeHealthCheckInfos == null) {
 341             result = false;
 342         } else {
 343             writeLogAlarm(prevBeHealthCheckInfos);
 344             result = true;
 345         }
 346         return result;
 347     }
 348
 349     private void writeLogAlarm(List<HealthCheckInfo> prevBeHealthCheckInfos) {
 350         logAlarm(format(COMPONENT_CHANGED_MESSAGE, "", prevBeHealthCheckInfos == null ? "null" : "true",
 351             prevBeHealthCheckInfos == null ? "true" : "null"));
 352     }
 353
 354     private boolean isHealthStatusChanged(boolean result, Map<String, HealthCheckStatus> currentValues, Map<String, HealthCheckStatus> prevValues) {
 355         for (Entry<String, HealthCheckStatus> entry : currentValues.entrySet()) {
 356             String key = entry.getKey();
 357             HealthCheckStatus value = entry.getValue();
 358             if (!prevValues.containsKey(key)) {
 359                 result = true; //component missing
 360                 logAlarm(format(COMPONENT_CHANGED_MESSAGE, key, prevValues.get(key), currentValues.get(key)));
 361                 break;
 362             }
 363             HealthCheckStatus prevHealthCheckStatus = prevValues.get(key);
 364             if (value != prevHealthCheckStatus) {
 365                 result = true; //component status changed
 366                 logAlarm(format(COMPONENT_CHANGED_MESSAGE, key, prevValues.get(key), currentValues.get(key)));
 367                 break;
 368             }
 369         }
 370         return result;
 371     }
 372
 373     private void updateHealthCheckStatusMap(Map<String, HealthCheckStatus> currentValues, Map<String, HealthCheckStatus> prevValues) {
 374         Map<String, HealthCheckStatus> notPresent;
 375         if (currentValues.keySet().containsAll(prevValues.keySet())) {
 376             notPresent = new HashMap<>(currentValues);
 377             notPresent.keySet().removeAll(prevValues.keySet());
 378         } else {
 379             notPresent = new HashMap<>(prevValues);
 380             notPresent.keySet().removeAll(currentValues.keySet());
 381         }
 382         for (String component : notPresent.keySet()) {
 383             logAlarm(format(COMPONENT_CHANGED_MESSAGE, component, prevValues.get(component), currentValues.get(component)));
 384         }
 385     }
 386
 387     @VisibleForTesting
 388     String buildOnBoardingHealthCheckUrl() {
 389         Configuration.OnboardingConfig onboardingConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getOnboarding();
 390         if (onboardingConfig != null) {
 391             return String.format(hcUrl, onboardingConfig.getProtocol(), onboardingConfig.getHost(), onboardingConfig.getPort(),
 392                 onboardingConfig.getHealthCheckUri());
 393         }
 394         log.error("Onboarding health check configuration is missing.");
 395         return null;
 396     }
 397
 398     public class HealthCheckScheduledTask implements Runnable {
 399
 400         @Override
 401         public void run() {
 402             mdcFieldsHandler.addInfoForErrorAndDebugLogging(LOG_PARTNER_NAME);
 403             Configuration config = ConfigurationManager.getConfigurationManager().getConfiguration();
 404             log.trace("Executing BE Health Check Task");
 405             List<HealthCheckInfo> currentBeHealthCheckInfos = getBeHealthCheckInfos();
 406             boolean healthStatus = healthCheckUtil.getAggregateStatus(currentBeHealthCheckInfos, config.getHealthStatusExclude());
 407             boolean prevHealthStatus = healthCheckUtil.getAggregateStatus(prevBeHealthCheckInfos, config.getHealthStatusExclude());
 408             boolean anyStatusChanged = anyStatusChanged(currentBeHealthCheckInfos, prevBeHealthCheckInfos);
 409             if (prevHealthStatus != healthStatus || anyStatusChanged) {
 410                 log.trace("BE Health State Changed to {}. Issuing alarm / recovery alarm...", healthStatus);
 411                 prevBeHealthCheckInfos = currentBeHealthCheckInfos;
 412                 logAlarm(healthStatus);
 413             }
 414         }
 415
 416         private void logAlarm(boolean prevHealthState) {
 417             if (prevHealthState) {
 418                 BeEcompErrorManager.getInstance().logBeHealthCheckRecovery(BE_HEALTH_CHECK_STR);
 419             } else {
 420                 BeEcompErrorManager.getInstance().logBeHealthCheckError(BE_HEALTH_CHECK_STR);
 421             }
 422         }
 423     }
 424 }