catalog-be/src/main/java/org/openecomp/sdc/be/components/health/HealthCheckBusinessLogic.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.be.components.health;
  22
  23 import com.fasterxml.jackson.core.type.TypeReference;
  24 import com.fasterxml.jackson.databind.ObjectMapper;
  25 import com.google.common.annotations.VisibleForTesting;
  26 import org.apache.commons.lang3.tuple.ImmutablePair;
  27 import org.apache.commons.lang3.tuple.Pair;
  28 import org.openecomp.sdc.be.catalog.impl.DmaapProducerHealth;
  29 import org.openecomp.sdc.be.components.distribution.engine.DistributionEngineClusterHealth;
  30 import org.openecomp.sdc.be.components.distribution.engine.DmaapHealth;
  31 import org.openecomp.sdc.be.components.impl.CADIHealthCheck;
  32 import org.openecomp.sdc.be.components.impl.CassandraHealthCheck;
  33 import org.openecomp.sdc.be.config.BeEcompErrorManager;
  34 import org.openecomp.sdc.be.config.Configuration;
  35 import org.openecomp.sdc.be.config.ConfigurationManager;
  36 import org.openecomp.sdc.be.dao.janusgraph.JanusGraphGenericDao;
  37 import org.openecomp.sdc.be.switchover.detector.SwitchoverDetector;
  38 import org.openecomp.sdc.common.api.HealthCheckInfo;
  39 import org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus;
  40 import org.openecomp.sdc.common.http.client.api.HttpRequest;
  41 import org.openecomp.sdc.common.http.client.api.HttpResponse;
  42 import org.openecomp.sdc.common.http.config.HttpClientConfig;
  43 import org.openecomp.sdc.common.http.config.Timeouts;
  44 import org.openecomp.sdc.common.log.elements.LogFieldsMdcHandler;
  45 import org.openecomp.sdc.common.log.wrappers.Logger;
  46 import org.openecomp.sdc.common.util.HealthCheckUtil;
  47 import org.springframework.beans.factory.annotation.Autowired;
  48 import org.springframework.stereotype.Component;
  49
  50 import javax.annotation.PostConstruct;
  51 import javax.annotation.PreDestroy;
  52 import javax.annotation.Resource;
  53 import java.util.ArrayList;
  54 import java.util.HashMap;
  55 import java.util.List;
  56 import java.util.Map;
  57 import java.util.Map.Entry;
  58 import java.util.concurrent.ScheduledExecutorService;
  59 import java.util.concurrent.ScheduledFuture;
  60 import java.util.concurrent.TimeUnit;
  61 import java.util.stream.Collectors;
  62
  63 import static java.lang.String.format;
  64 import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
  65 import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
  66 import static org.apache.http.HttpStatus.SC_OK;
  67 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_BE;
  68 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_CASSANDRA;
  69 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_DMAAP_PRODUCER;
  70 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_ECOMP_PORTAL;
  71 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_JANUSGRAPH;
  72 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_ON_BOARDING;
  73 import static org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus.DOWN;
  74 import static org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus.UP;
  75 import static org.openecomp.sdc.common.impl.ExternalConfiguration.getAppVersion;
  76
  77
  78 @Component("healthCheckBusinessLogic")
  79 public class HealthCheckBusinessLogic {
  80
  81     private static String hcUrl = "%s://%s:%s%s";
  82     private static final String BE_HEALTH_CHECK_STR = "beHealthCheck";
  83     private static final String LOG_PARTNER_NAME = "SDC.BE";
  84     private static final String COMPONENT_CHANGED_MESSAGE = "BE Component %s state changed from %s to %s";
  85     private static final Logger log = Logger.getLogger(HealthCheckBusinessLogic.class.getName());
  86     private static final HealthCheckUtil healthCheckUtil = new HealthCheckUtil();
  87     private final ScheduledExecutorService healthCheckScheduler = newSingleThreadScheduledExecutor((Runnable r) -> new Thread(r, "BE-Health-Check-Task"));
  88     private HealthCheckScheduledTask healthCheckScheduledTask = null;
  89     private static LogFieldsMdcHandler mdcFieldsHandler = new LogFieldsMdcHandler();
  90
  91     @Resource
  92     private JanusGraphGenericDao janusGraphGenericDao;
  93     @Resource
  94     private DistributionEngineClusterHealth distributionEngineClusterHealth;
  95     @Resource
  96     private DmaapHealth dmaapHealth;
  97     @Resource
  98     private DmaapProducerHealth dmaapProducerHealth;
  99     @Resource
 100     private CassandraHealthCheck cassandraHealthCheck;
 101     @Resource
 102     private PortalHealthCheckBuilder portalHealthCheck;
 103
 104     @Autowired
 105     private SwitchoverDetector switchoverDetector;
 106     private volatile List<HealthCheckInfo> prevBeHealthCheckInfos = null;
 107     private ScheduledFuture<?> scheduledFuture = null;
 108
 109     @PostConstruct
 110     public void init() {
 111
 112         prevBeHealthCheckInfos = getBeHealthCheckInfos();
 113
 114         log.debug("After initializing prevBeHealthCheckInfos: {}", prevBeHealthCheckInfos);
 115
 116         healthCheckScheduledTask = new HealthCheckScheduledTask();
 117
 118         if (this.scheduledFuture == null) {
 119             this.scheduledFuture = this.healthCheckScheduler.scheduleAtFixedRate(healthCheckScheduledTask, 0, 3, TimeUnit.SECONDS);
 120         }
 121
 122     }
 123
 124     public boolean isDistributionEngineUp() {
 125
 126         HealthCheckInfo healthCheckInfo = distributionEngineClusterHealth.getHealthCheckInfo();
 127         return healthCheckInfo.getHealthCheckStatus() != DOWN;
 128     }
 129
 130     public Pair<Boolean, List<HealthCheckInfo>> getBeHealthCheckInfosStatus() {
 131         Configuration config = ConfigurationManager.getConfigurationManager().getConfiguration();
 132         return new ImmutablePair<>(healthCheckUtil.getAggregateStatus(prevBeHealthCheckInfos, config.getHealthStatusExclude()), prevBeHealthCheckInfos);
 133     }
 134
 135     private List<HealthCheckInfo> getBeHealthCheckInfos() {
 136
 137         log.trace("In getBeHealthCheckInfos");
 138
 139         List<HealthCheckInfo> healthCheckInfos = new ArrayList<>();
 140
 141         //Dmaap
 142         HealthCheckInfo info;
 143         if ((info = getDmaapHealthCheck()) != null) {
 144             healthCheckInfos.add(info);
 145         }
 146
 147         //DmaapProducer
 148         healthCheckInfos.add(getDmaapProducerHealthCheck());
 149
 150         // BE
 151         healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_BE, UP, getAppVersion(), "OK"));
 152
 153         // JanusGraph
 154         healthCheckInfos.add(getJanusGraphHealthCheck());
 155
 156         // Distribution Engine
 157         healthCheckInfos.add(distributionEngineClusterHealth.getHealthCheckInfo());
 158
 159         //Cassandra
 160         healthCheckInfos.add(getCassandraHealthCheck());
 161
 162         // Amdocs
 163         healthCheckInfos.add(getHostedComponentsBeHealthCheck(HC_COMPONENT_ON_BOARDING, buildOnBoardingHealthCheckUrl()));
 164
 165          //ECOMP Portal
 166         healthCheckInfos.add(portalHealthCheck.getHealthCheckInfo());
 167
 168         //CADI
 169         healthCheckInfos.add(CADIHealthCheck.getCADIHealthCheckInstance().getCADIStatus());
 170
 171         return healthCheckInfos;
 172     }
 173
 174     private HealthCheckInfo getDmaapHealthCheck() {
 175         HealthCheckInfo healthCheckInfo = null;
 176         if(ConfigurationManager.getConfigurationManager().getConfiguration().getDmaapConsumerConfiguration().isActive()){
 177             String appVersion = getAppVersion();
 178             dmaapHealth.getHealthCheckInfo().setVersion(appVersion);
 179             healthCheckInfo = dmaapHealth.getHealthCheckInfo();
 180         } else {
 181           log.debug("Dmaap health check disabled");
 182         }
 183         return healthCheckInfo;
 184     }
 185
 186     private HealthCheckInfo getDmaapProducerHealthCheck() {
 187         if (ConfigurationManager.getConfigurationManager().getConfiguration().getDmaapConsumerConfiguration().isActive()) {
 188             String appVersion = getAppVersion();
 189             dmaapProducerHealth.getHealthCheckInfo().setVersion(appVersion);
 190             return dmaapProducerHealth.getHealthCheckInfo();
 191         } else {
 192             log.debug("Dmaap health check disabled");
 193             String description = ("Dmaap health check disabled");
 194             return new HealthCheckInfo(HC_COMPONENT_DMAAP_PRODUCER, DOWN, null, description);
 195         }
 196     }
 197
 198     public HealthCheckInfo getJanusGraphHealthCheck() {
 199         // JanusGraph health check and version
 200         String description;
 201         boolean isJanusGraphUp;
 202         HealthCheckInfo healthCheckInfo = new HealthCheckInfo(HC_COMPONENT_JANUSGRAPH, DOWN, null, null);
 203
 204         try {
 205             isJanusGraphUp = janusGraphGenericDao.isGraphOpen();
 206         } catch (Exception e) {
 207             description = "JanusGraph error: " + e.getMessage();
 208             healthCheckInfo.setDescription(description);
 209             log.error(description);
 210             return healthCheckInfo;
 211         }
 212         if (isJanusGraphUp) {
 213             description = "OK";
 214             healthCheckInfo.setDescription(description);
 215             healthCheckInfo.setHealthCheckStatus(HealthCheckInfo.HealthCheckStatus.UP);
 216
 217         } else {
 218             description = "JanusGraph graph is down";
 219             healthCheckInfo.setDescription(description);
 220         }
 221         return healthCheckInfo;
 222     }
 223
 224     private HealthCheckInfo getCassandraHealthCheck() {
 225
 226         String description;
 227         boolean isCassandraUp = false;
 228         HealthCheckInfo healthCheckInfo = new HealthCheckInfo(HC_COMPONENT_CASSANDRA, DOWN, null, null);
 229
 230         try {
 231             isCassandraUp = cassandraHealthCheck.getCassandraStatus();
 232         } catch (Exception e) {
 233             description = "Cassandra error: " + e.getMessage();
 234             log.error(description, e);
 235         }
 236         if (isCassandraUp) {
 237             description = "OK";
 238 //            healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_CASSANDRA, UP, null, description));
 239             healthCheckInfo.setHealthCheckStatus(HealthCheckStatus.UP);
 240             healthCheckInfo.setDescription(description);
 241         } else {
 242             description = "Cassandra is down";
 243 //            healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_CASSANDRA, DOWN, null, description));
 244             healthCheckInfo.setDescription(description);
 245         }
 246         return healthCheckInfo;
 247     }
 248
 249     private HealthCheckInfo getHostedComponentsBeHealthCheck(String componentName, String healthCheckUrl) {
 250         HealthCheckStatus healthCheckStatus;
 251         String description;
 252         String version = null;
 253         List<HealthCheckInfo> componentsInfo = new ArrayList<>();
 254         final int timeout = 3000;
 255
 256         if (healthCheckUrl != null) {
 257             try {
 258                 HttpResponse<String> httpResponse = HttpRequest.get(healthCheckUrl, new HttpClientConfig(new Timeouts(timeout, timeout)));
 259                 int statusCode = httpResponse.getStatusCode();
 260                 String aggDescription = "";
 261
 262                 if ((statusCode == SC_OK || statusCode == SC_INTERNAL_SERVER_ERROR) && !componentName.equals(HC_COMPONENT_ECOMP_PORTAL)) {
 263                     String response = httpResponse.getResponse();
 264                     log.trace("{} Health Check response: {}", componentName, response);
 265                     ObjectMapper mapper = new ObjectMapper();
 266                     Map<String, Object> healthCheckMap = mapper.readValue(response, new TypeReference<Map<String, Object>>() {
 267                     });
 268                     version = getVersion(healthCheckMap);
 269                     if (healthCheckMap.containsKey("componentsInfo")) {
 270                         componentsInfo = mapper.convertValue(healthCheckMap.get("componentsInfo"), new TypeReference<List<HealthCheckInfo>>() {
 271                         });
 272                     }
 273                     aggDescription = getAggDescription(componentsInfo, aggDescription);
 274                 } else {
 275                     log.trace("{} Health Check Response code: {}", componentName, statusCode);
 276                 }
 277
 278                 if (statusCode != SC_OK) {
 279                     healthCheckStatus = DOWN;
 280                     description = getDescription(componentName, aggDescription);
 281                     setDescriptionToObject(description, componentsInfo);
 282                 } else {
 283                     healthCheckStatus = UP;
 284                     description = "OK";
 285                 }
 286
 287             } catch (Exception e) {
 288                 log.error("{} unexpected response: ", componentName, e);
 289                 healthCheckStatus = DOWN;
 290                 description = componentName + " unexpected response: " + e.getMessage();
 291                 addToHealthCheckInfoObject(description, componentsInfo);
 292             }
 293         } else {
 294             healthCheckStatus = DOWN;
 295             description = componentName + " health check Configuration is missing";
 296             componentsInfo.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, description));
 297         }
 298         return new HealthCheckInfo(componentName, healthCheckStatus, version, description, componentsInfo);
 299     }
 300
 301     private void addToHealthCheckInfoObject(String description, List<HealthCheckInfo> componentsInfo) {
 302         if (componentsInfo != null && componentsInfo.isEmpty()) {
 303             componentsInfo.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, description));
 304         }
 305     }
 306
 307     private void setDescriptionToObject(String description, List<HealthCheckInfo> componentsInfo) {
 308         if (componentsInfo.isEmpty()) {
 309             componentsInfo.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, description));
 310         }
 311     }
 312
 313     private String getDescription(String componentName, String aggDescription) {
 314         String description;
 315         description = aggDescription.length() > 0
 316                 ? aggDescription
 317                 : componentName + " is Down, specific reason unknown";//No inner component returned DOWN, but the status of HC is still DOWN.
 318         return description;
 319     }
 320
 321     private String getVersion(Map<String, Object> healthCheckMap) {
 322         return healthCheckMap.get("sdcVersion") != null ? healthCheckMap.get("sdcVersion").toString() : null;
 323     }
 324
 325     private String getAggDescription(List<HealthCheckInfo> componentsInfo, String aggDescription) {
 326         if (!componentsInfo.isEmpty()) {
 327             aggDescription = healthCheckUtil.getAggregateDescription(componentsInfo);
 328         } else {
 329             componentsInfo.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, null));
 330         }
 331         return aggDescription;
 332     }
 333
 334     @PreDestroy
 335     protected void destroy() {
 336
 337         if (scheduledFuture != null) {
 338             scheduledFuture.cancel(true);
 339             scheduledFuture = null;
 340         }
 341
 342         if (healthCheckScheduler != null) {
 343             healthCheckScheduler.shutdown();
 344         }
 345
 346     }
 347
 348     private void logAlarm(String componentChangedMsg) {
 349         BeEcompErrorManager.getInstance().logBeHealthCheckRecovery(componentChangedMsg);
 350     }
 351
 352     public String getSiteMode() {
 353         return switchoverDetector.getSiteMode();
 354     }
 355
 356     public boolean anyStatusChanged(List<HealthCheckInfo> beHealthCheckInfos, List<HealthCheckInfo> prevBeHealthCheckInfos) {
 357
 358         boolean result = false;
 359
 360         if (beHealthCheckInfos != null && prevBeHealthCheckInfos != null) {
 361
 362             Map<String, HealthCheckStatus> currentValues = beHealthCheckInfos.stream().collect(Collectors.toMap(HealthCheckInfo::getHealthCheckComponent, HealthCheckInfo::getHealthCheckStatus));
 363             Map<String, HealthCheckStatus> prevValues = prevBeHealthCheckInfos.stream().collect(Collectors.toMap(HealthCheckInfo::getHealthCheckComponent, HealthCheckInfo::getHealthCheckStatus));
 364
 365             if (currentValues != null && prevValues != null) {
 366                 int currentSize = currentValues.size();
 367                 int prevSize = prevValues.size();
 368
 369                 if (currentSize != prevSize) {
 370                     result = true; //extra/missing component
 371                     updateHealthCheckStatusMap(currentValues, prevValues);
 372                 } else {
 373                     result = isHealthStatusChanged(result, currentValues, prevValues);
 374                 }
 375             }
 376
 377         } else if (beHealthCheckInfos == null && prevBeHealthCheckInfos == null) {
 378             result = false;
 379         } else {
 380             writeLogAlarm(prevBeHealthCheckInfos);
 381             result = true;
 382         }
 383
 384         return result;
 385     }
 386
 387     private void writeLogAlarm(List<HealthCheckInfo> prevBeHealthCheckInfos) {
 388         logAlarm(format(COMPONENT_CHANGED_MESSAGE, "", prevBeHealthCheckInfos == null ? "null" : "true", prevBeHealthCheckInfos == null ? "true" : "null"));
 389     }
 390
 391     private boolean isHealthStatusChanged(boolean result, Map<String, HealthCheckStatus> currentValues, Map<String, HealthCheckStatus> prevValues) {
 392         for (Entry<String, HealthCheckStatus> entry : currentValues.entrySet()) {
 393             String key = entry.getKey();
 394             HealthCheckStatus value = entry.getValue();
 395
 396             if (!prevValues.containsKey(key)) {
 397                 result = true; //component missing
 398                 logAlarm(format(COMPONENT_CHANGED_MESSAGE, key, prevValues.get(key), currentValues.get(key)));
 399                 break;
 400             }
 401
 402             HealthCheckStatus prevHealthCheckStatus = prevValues.get(key);
 403
 404             if (value != prevHealthCheckStatus) {
 405                 result = true; //component status changed
 406                 logAlarm(format(COMPONENT_CHANGED_MESSAGE, key, prevValues.get(key), currentValues.get(key)));
 407                 break;
 408             }
 409         }
 410         return result;
 411     }
 412
 413     private void updateHealthCheckStatusMap(Map<String, HealthCheckStatus> currentValues, Map<String, HealthCheckStatus> prevValues) {
 414         Map<String, HealthCheckStatus> notPresent;
 415         if (currentValues.keySet().containsAll(prevValues.keySet())) {
 416             notPresent = new HashMap<>(currentValues);
 417             notPresent.keySet().removeAll(prevValues.keySet());
 418         } else {
 419             notPresent = new HashMap<>(prevValues);
 420             notPresent.keySet().removeAll(currentValues.keySet());
 421         }
 422
 423         for (String component : notPresent.keySet()) {
 424             logAlarm(format(COMPONENT_CHANGED_MESSAGE, component, prevValues.get(component), currentValues.get(component)));
 425         }
 426     }
 427     @VisibleForTesting
 428     String buildOnBoardingHealthCheckUrl() {
 429
 430         Configuration.OnboardingConfig onboardingConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getOnboarding();
 431         if (onboardingConfig != null) {
 432             return String.format(hcUrl, onboardingConfig.getProtocol(), onboardingConfig.getHost(),
 433                     onboardingConfig.getPort(),onboardingConfig.getHealthCheckUri());
 434         }
 435         log.error("Onboarding health check configuration is missing.");
 436         return null;
 437     }
 438
 439     public class HealthCheckScheduledTask implements Runnable {
 440         @Override
 441         public void run() {
 442             mdcFieldsHandler.addInfoForErrorAndDebugLogging(LOG_PARTNER_NAME);
 443             Configuration config = ConfigurationManager.getConfigurationManager().getConfiguration();
 444             log.trace("Executing BE Health Check Task");
 445
 446             List<HealthCheckInfo> currentBeHealthCheckInfos = getBeHealthCheckInfos();
 447             boolean healthStatus = healthCheckUtil.getAggregateStatus(currentBeHealthCheckInfos, config.getHealthStatusExclude());
 448
 449             boolean prevHealthStatus = healthCheckUtil.getAggregateStatus(prevBeHealthCheckInfos, config.getHealthStatusExclude());
 450
 451             boolean anyStatusChanged = anyStatusChanged(currentBeHealthCheckInfos, prevBeHealthCheckInfos);
 452
 453             if (prevHealthStatus != healthStatus || anyStatusChanged) {
 454                 log.trace("BE Health State Changed to {}. Issuing alarm / recovery alarm...", healthStatus);
 455
 456                 prevBeHealthCheckInfos = currentBeHealthCheckInfos;
 457                 logAlarm(healthStatus);
 458             }
 459         }
 460
 461         private void logAlarm(boolean prevHealthState) {
 462             if (prevHealthState) {
 463                 BeEcompErrorManager.getInstance().logBeHealthCheckRecovery(BE_HEALTH_CHECK_STR);
 464             } else {
 465                 BeEcompErrorManager.getInstance().logBeHealthCheckError(BE_HEALTH_CHECK_STR);
 466             }
 467         }
 468
 469
 470     }
 471
 472 }