catalog-be/src/main/java/org/openecomp/sdc/be/components/health/HealthCheckBusinessLogic.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.be.components.health;
  22
  23 import com.fasterxml.jackson.core.type.TypeReference;
  24 import com.fasterxml.jackson.databind.ObjectMapper;
  25 import org.apache.commons.lang3.tuple.ImmutablePair;
  26 import org.apache.commons.lang3.tuple.Pair;
  27 import org.openecomp.sdc.be.components.distribution.engine.DistributionEngineClusterHealth;
  28 import org.openecomp.sdc.be.components.distribution.engine.DmaapHealth;
  29 import org.openecomp.sdc.be.components.impl.CassandraHealthCheck;
  30 import org.openecomp.sdc.be.config.BeEcompErrorManager;
  31 import org.openecomp.sdc.be.config.Configuration;
  32 import org.openecomp.sdc.be.config.ConfigurationManager;
  33 import org.openecomp.sdc.be.dao.impl.EsHealthCheckDao;
  34 import org.openecomp.sdc.be.dao.janusgraph.JanusGraphGenericDao;
  35 import org.openecomp.sdc.be.switchover.detector.SwitchoverDetector;
  36 import org.openecomp.sdc.common.api.HealthCheckInfo;
  37 import org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus;
  38 import org.openecomp.sdc.common.http.client.api.HttpRequest;
  39 import org.openecomp.sdc.common.http.client.api.HttpResponse;
  40 import org.openecomp.sdc.common.http.config.HttpClientConfig;
  41 import org.openecomp.sdc.common.http.config.Timeouts;
  42 import org.openecomp.sdc.common.log.wrappers.Logger;
  43 import org.openecomp.sdc.common.util.HealthCheckUtil;
  44 import org.springframework.beans.factory.annotation.Autowired;
  45 import org.springframework.stereotype.Component;
  46
  47 import javax.annotation.PostConstruct;
  48 import javax.annotation.PreDestroy;
  49 import javax.annotation.Resource;
  50 import java.util.ArrayList;
  51 import java.util.HashMap;
  52 import java.util.List;
  53 import java.util.Map;
  54 import java.util.Map.Entry;
  55 import java.util.concurrent.ScheduledExecutorService;
  56 import java.util.concurrent.ScheduledFuture;
  57 import java.util.concurrent.TimeUnit;
  58 import java.util.stream.Collectors;
  59
  60 import static java.lang.String.format;
  61 import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
  62 import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
  63 import static org.apache.http.HttpStatus.SC_OK;
  64 import static org.openecomp.sdc.common.api.Constants.*;
  65 import static org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus.DOWN;
  66 import static org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus.UP;
  67 import static org.openecomp.sdc.common.impl.ExternalConfiguration.getAppVersion;
  68
  69
  70 @Component("healthCheckBusinessLogic")
  71 public class HealthCheckBusinessLogic {
  72
  73     protected static final String BE_HEALTH_LOG_CONTEXT = "be.healthcheck";
  74     private static final String BE_HEALTH_CHECK_STR = "beHealthCheck";
  75     private static final String COMPONENT_CHANGED_MESSAGE = "BE Component %s state changed from %s to %s";
  76     private static final Logger log = Logger.getLogger(HealthCheckBusinessLogic.class.getName());
  77     private static final HealthCheckUtil healthCheckUtil = new HealthCheckUtil();
  78     private final ScheduledExecutorService healthCheckScheduler = newSingleThreadScheduledExecutor((Runnable r) -> new Thread(r, "BE-Health-Check-Task"));
  79     private HealthCheckScheduledTask healthCheckScheduledTask = null;
  80     @Resource
  81     private JanusGraphGenericDao janusGraphGenericDao;
  82     @Resource
  83     private EsHealthCheckDao esHealthCheckDao;
  84     @Resource
  85     private DistributionEngineClusterHealth distributionEngineClusterHealth;
  86     @Resource
  87     private DmaapHealth dmaapHealth;
  88     @Resource
  89     private CassandraHealthCheck cassandraHealthCheck;
  90     @Autowired
  91     private SwitchoverDetector switchoverDetector;
  92     private volatile List<HealthCheckInfo> prevBeHealthCheckInfos = null;
  93     private ScheduledFuture<?> scheduledFuture = null;
  94
  95     @PostConstruct
  96     public void init() {
  97
  98         prevBeHealthCheckInfos = getBeHealthCheckInfos();
  99
 100         log.debug("After initializing prevBeHealthCheckInfos: {}", prevBeHealthCheckInfos);
 101
 102         healthCheckScheduledTask = new HealthCheckScheduledTask();
 103
 104         if (this.scheduledFuture == null) {
 105             this.scheduledFuture = this.healthCheckScheduler.scheduleAtFixedRate(healthCheckScheduledTask, 0, 3, TimeUnit.SECONDS);
 106         }
 107
 108     }
 109
 110     public boolean isDistributionEngineUp() {
 111
 112         HealthCheckInfo healthCheckInfo = distributionEngineClusterHealth.getHealthCheckInfo();
 113         return !healthCheckInfo.getHealthCheckStatus().equals(DOWN);
 114     }
 115
 116     public Pair<Boolean, List<HealthCheckInfo>> getBeHealthCheckInfosStatus() {
 117         Configuration config = ConfigurationManager.getConfigurationManager().getConfiguration();
 118         return new ImmutablePair<>(healthCheckUtil.getAggregateStatus(prevBeHealthCheckInfos, config.getHealthStatusExclude()), prevBeHealthCheckInfos);
 119     }
 120
 121     private List<HealthCheckInfo> getBeHealthCheckInfos() {
 122
 123         log.trace("In getBeHealthCheckInfos");
 124
 125         List<HealthCheckInfo> healthCheckInfos = new ArrayList<>();
 126
 127         //Dmaap
 128         getDmaapHealthCheck(healthCheckInfos);
 129
 130         // BE
 131         getBeHealthCheck(healthCheckInfos);
 132
 133         // JanusGraph
 134         getJanusGraphHealthCheck(healthCheckInfos);
 135         // ES
 136         getEsHealthCheck(healthCheckInfos);
 137
 138         // Distribution Engine
 139         getDistributionEngineCheck(healthCheckInfos);
 140
 141         //Cassandra
 142         getCassandraHealthCheck(healthCheckInfos);
 143
 144         // Amdocs
 145         getAmdocsHealthCheck(healthCheckInfos);
 146
 147         //DCAE
 148         getDcaeHealthCheck(healthCheckInfos);
 149
 150         return healthCheckInfos;
 151     }
 152
 153     private List<HealthCheckInfo> getEsHealthCheck(List<HealthCheckInfo> healthCheckInfos) {
 154
 155         // ES health check and version
 156         String appVersion = getAppVersion();
 157         HealthCheckStatus healthCheckStatus;
 158         String description;
 159
 160         try {
 161             healthCheckStatus = esHealthCheckDao.getClusterHealthStatus();
 162         } catch (Exception e) {
 163             healthCheckStatus = DOWN;
 164             description = "ES cluster error: " + e.getMessage();
 165             healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_ES, healthCheckStatus, appVersion, description));
 166             log.error(description, e);
 167             return healthCheckInfos;
 168         }
 169         if (healthCheckStatus.equals(DOWN)) {
 170             description = "ES cluster is down";
 171         } else {
 172             description = "OK";
 173         }
 174         healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_ES, healthCheckStatus, appVersion, description));
 175         return healthCheckInfos;
 176     }
 177
 178     private List<HealthCheckInfo> getBeHealthCheck(List<HealthCheckInfo> healthCheckInfos) {
 179         String appVersion = getAppVersion();
 180         String description = "OK";
 181         healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_BE, UP, appVersion, description));
 182         return healthCheckInfos;
 183     }
 184
 185     private List<HealthCheckInfo> getDmaapHealthCheck(List<HealthCheckInfo> healthCheckInfos) {
 186         if(ConfigurationManager.getConfigurationManager().getConfiguration().getDmaapConsumerConfiguration().isActive()){
 187             String appVersion = getAppVersion();
 188             dmaapHealth.getHealthCheckInfo().setVersion(appVersion);
 189             healthCheckInfos.add(dmaapHealth.getHealthCheckInfo());
 190         } else {
 191           log.debug("Dmaap health check disabled");
 192         }
 193
 194         return healthCheckInfos;
 195     }
 196
 197
 198     public List<HealthCheckInfo> getJanusGraphHealthCheck(List<HealthCheckInfo> healthCheckInfos) {
 199         // JanusGraph health check and version
 200         String description;
 201         boolean isJanusGraphUp;
 202
 203         try {
 204             isJanusGraphUp = janusGraphGenericDao.isGraphOpen();
 205         } catch (Exception e) {
 206             description = "JanusGraph error: ";
 207             healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_JANUSGRAPH, DOWN, null, description));
 208             log.error(description, e);
 209             return healthCheckInfos;
 210         }
 211         if (isJanusGraphUp) {
 212             description = "OK";
 213             healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_JANUSGRAPH, UP, null, description));
 214         } else {
 215             description = "JanusGraph graph is down";
 216             healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_JANUSGRAPH, DOWN, null, description));
 217         }
 218         return healthCheckInfos;
 219     }
 220
 221     private List<HealthCheckInfo> getCassandraHealthCheck(List<HealthCheckInfo> healthCheckInfos) {
 222
 223         String description;
 224         boolean isCassandraUp = false;
 225
 226         try {
 227             isCassandraUp = cassandraHealthCheck.getCassandraStatus();
 228         } catch (Exception e) {
 229             description = "Cassandra error: " + e.getMessage();
 230             log.error(description, e);
 231         }
 232         if (isCassandraUp) {
 233             description = "OK";
 234             healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_CASSANDRA, UP, null, description));
 235         } else {
 236             description = "Cassandra is down";
 237             healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_CASSANDRA, DOWN, null, description));
 238         }
 239         return healthCheckInfos;
 240
 241     }
 242
 243     private void getDistributionEngineCheck(List<HealthCheckInfo> healthCheckInfos) {
 244
 245         HealthCheckInfo healthCheckInfo = distributionEngineClusterHealth.getHealthCheckInfo();
 246
 247         healthCheckInfos.add(healthCheckInfo);
 248
 249     }
 250
 251     private List<HealthCheckInfo> getAmdocsHealthCheck(List<HealthCheckInfo> healthCheckInfos) {
 252         HealthCheckInfo beHealthCheckInfo = getHostedComponentsBeHealthCheck(HC_COMPONENT_ON_BOARDING, buildOnBoardingHealthCheckUrl());
 253         healthCheckInfos.add(beHealthCheckInfo);
 254         return healthCheckInfos;
 255     }
 256
 257     private List<HealthCheckInfo> getDcaeHealthCheck(List<HealthCheckInfo> healthCheckInfos) {
 258         HealthCheckInfo beHealthCheckInfo = getHostedComponentsBeHealthCheck(HC_COMPONENT_DCAE, buildDcaeHealthCheckUrl());
 259         healthCheckInfos.add(beHealthCheckInfo);
 260         return healthCheckInfos;
 261     }
 262
 263     private HealthCheckInfo getHostedComponentsBeHealthCheck(String componentName, String healthCheckUrl) {
 264         HealthCheckStatus healthCheckStatus;
 265         String description;
 266         String version = null;
 267         List<HealthCheckInfo> componentsInfo = new ArrayList<>();
 268         final int timeout = 3000;
 269
 270         if (healthCheckUrl != null) {
 271             try {
 272                 HttpResponse<String> httpResponse = HttpRequest.get(healthCheckUrl, new HttpClientConfig(new Timeouts(timeout, timeout)));
 273                 int statusCode = httpResponse.getStatusCode();
 274                 String aggDescription = "";
 275
 276                 if (statusCode == SC_OK || statusCode == SC_INTERNAL_SERVER_ERROR) {
 277                     String response = httpResponse.getResponse();
 278                     log.trace("{} Health Check response: {}", componentName, response);
 279                     ObjectMapper mapper = new ObjectMapper();
 280                     Map<String, Object> healthCheckMap = mapper.readValue(response, new TypeReference<Map<String, Object>>() {
 281                     });
 282                     version = healthCheckMap.get("sdcVersion") != null ? healthCheckMap.get("sdcVersion").toString() : null;
 283                     if (healthCheckMap.containsKey("componentsInfo")) {
 284                         componentsInfo = mapper.convertValue(healthCheckMap.get("componentsInfo"), new TypeReference<List<HealthCheckInfo>>() {
 285                         });
 286                     }
 287
 288                     if (!componentsInfo.isEmpty()) {
 289                         aggDescription = healthCheckUtil.getAggregateDescription(componentsInfo, null);
 290                     } else {
 291                         componentsInfo.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, null));
 292                     }
 293                 } else {
 294                     log.trace("{} Health Check Response code: {}", componentName, statusCode);
 295                 }
 296
 297                 if (statusCode != SC_OK) {
 298                     healthCheckStatus = DOWN;
 299                     description = aggDescription.length() > 0
 300                             ? aggDescription
 301                             : componentName + " is Down, specific reason unknown";//No inner component returned DOWN, but the status of HC is still DOWN.
 302                     if (componentsInfo.isEmpty()) {
 303                         componentsInfo.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, description));
 304                     }
 305                 } else {
 306                     healthCheckStatus = UP;
 307                     description = "OK";
 308                 }
 309
 310             } catch (Exception e) {
 311                 log.error("{} unexpected response: ", componentName, e);
 312                 healthCheckStatus = DOWN;
 313                 description = componentName + " unexpected response: " + e.getMessage();
 314                 if (componentsInfo != null && componentsInfo.isEmpty()) {
 315                     componentsInfo.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, description));
 316                 }
 317             }
 318         } else {
 319             healthCheckStatus = DOWN;
 320             description = componentName + " health check Configuration is missing";
 321             componentsInfo.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, description));
 322         }
 323
 324         return new HealthCheckInfo(componentName, healthCheckStatus, version, description, componentsInfo);
 325     }
 326
 327     @PreDestroy
 328     protected void destroy() {
 329
 330         if (scheduledFuture != null) {
 331             scheduledFuture.cancel(true);
 332             scheduledFuture = null;
 333         }
 334
 335         if (healthCheckScheduler != null) {
 336             healthCheckScheduler.shutdown();
 337         }
 338
 339     }
 340
 341     private void logAlarm(String componentChangedMsg) {
 342         BeEcompErrorManager.getInstance().logBeHealthCheckRecovery(componentChangedMsg);
 343     }
 344
 345     public String getSiteMode() {
 346         return switchoverDetector.getSiteMode();
 347     }
 348
 349     public boolean anyStatusChanged(List<HealthCheckInfo> beHealthCheckInfos, List<HealthCheckInfo> prevBeHealthCheckInfos) {
 350
 351         boolean result = false;
 352
 353         if (beHealthCheckInfos != null && prevBeHealthCheckInfos != null) {
 354
 355             Map<String, HealthCheckStatus> currentValues = beHealthCheckInfos.stream().collect(Collectors.toMap(HealthCheckInfo::getHealthCheckComponent, HealthCheckInfo::getHealthCheckStatus));
 356             Map<String, HealthCheckStatus> prevValues = prevBeHealthCheckInfos.stream().collect(Collectors.toMap(HealthCheckInfo::getHealthCheckComponent, HealthCheckInfo::getHealthCheckStatus));
 357
 358             if (currentValues != null && prevValues != null) {
 359                 int currentSize = currentValues.size();
 360                 int prevSize = prevValues.size();
 361
 362                 if (currentSize != prevSize) {
 363
 364                     result = true; //extra/missing component
 365
 366                     Map<String, HealthCheckStatus> notPresent = null;
 367                     if (currentValues.keySet().containsAll(prevValues.keySet())) {
 368                         notPresent = new HashMap<>(currentValues);
 369                         notPresent.keySet().removeAll(prevValues.keySet());
 370                     } else {
 371                         notPresent = new HashMap<>(prevValues);
 372                         notPresent.keySet().removeAll(currentValues.keySet());
 373                     }
 374
 375                     for (String component : notPresent.keySet()) {
 376                         logAlarm(format(COMPONENT_CHANGED_MESSAGE, component, prevValues.get(component), currentValues.get(component)));
 377                     }
 378
 379                 } else {
 380
 381                     for (Entry<String, HealthCheckStatus> entry : currentValues.entrySet()) {
 382                         String key = entry.getKey();
 383                         HealthCheckStatus value = entry.getValue();
 384
 385                         if (!prevValues.containsKey(key)) {
 386                             result = true; //component missing
 387                             logAlarm(format(COMPONENT_CHANGED_MESSAGE, key, prevValues.get(key), currentValues.get(key)));
 388                             break;
 389                         }
 390
 391                         HealthCheckStatus prevHealthCheckStatus = prevValues.get(key);
 392
 393                         if (value != prevHealthCheckStatus) {
 394                             result = true; //component status changed
 395                             logAlarm(format(COMPONENT_CHANGED_MESSAGE, key, prevValues.get(key), currentValues.get(key)));
 396                             break;
 397                         }
 398                     }
 399                 }
 400             }
 401
 402         } else if (beHealthCheckInfos == null && prevBeHealthCheckInfos == null) {
 403             result = false;
 404         } else {
 405             logAlarm(format(COMPONENT_CHANGED_MESSAGE, "", prevBeHealthCheckInfos == null ? "null" : "true", prevBeHealthCheckInfos == null ? "true" : "null"));
 406             result = true;
 407         }
 408
 409         return result;
 410     }
 411
 412     private String buildOnBoardingHealthCheckUrl() {
 413
 414         Configuration.OnboardingConfig onboardingConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getOnboarding();
 415
 416         if (onboardingConfig != null) {
 417             String protocol = onboardingConfig.getProtocol();
 418             String host = onboardingConfig.getHost();
 419             Integer port = onboardingConfig.getPort();
 420             String uri = onboardingConfig.getHealthCheckUri();
 421
 422             return protocol + "://" + host + ":" + port + uri;
 423         }
 424
 425         log.error("onboarding health check configuration is missing.");
 426         return null;
 427     }
 428
 429     private String buildDcaeHealthCheckUrl() {
 430
 431         Configuration.DcaeConfig dcaeConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getDcae();
 432
 433         if (dcaeConfig != null) {
 434             String protocol = dcaeConfig.getProtocol();
 435             String host = dcaeConfig.getHost();
 436             Integer port = dcaeConfig.getPort();
 437             String uri = dcaeConfig.getHealthCheckUri();
 438
 439             return protocol + "://" + host + ":" + port + uri;
 440         }
 441
 442         log.error("dcae health check configuration is missing.");
 443         return null;
 444     }
 445
 446     public class HealthCheckScheduledTask implements Runnable {
 447         @Override
 448         public void run() {
 449             Configuration config = ConfigurationManager.getConfigurationManager().getConfiguration();
 450             log.trace("Executing BE Health Check Task");
 451
 452             List<HealthCheckInfo> currentBeHealthCheckInfos = getBeHealthCheckInfos();
 453             boolean healthStatus = healthCheckUtil.getAggregateStatus(currentBeHealthCheckInfos, config.getHealthStatusExclude());
 454
 455             boolean prevHealthStatus = healthCheckUtil.getAggregateStatus(prevBeHealthCheckInfos, config.getHealthStatusExclude());
 456
 457             boolean anyStatusChanged = anyStatusChanged(currentBeHealthCheckInfos, prevBeHealthCheckInfos);
 458
 459             if (prevHealthStatus != healthStatus || anyStatusChanged) {
 460                 log.trace("BE Health State Changed to {}. Issuing alarm / recovery alarm...", healthStatus);
 461
 462                 prevBeHealthCheckInfos = currentBeHealthCheckInfos;
 463                 logAlarm(healthStatus);
 464             }
 465         }
 466
 467         private void logAlarm(boolean prevHealthState) {
 468             if (prevHealthState) {
 469                 BeEcompErrorManager.getInstance().logBeHealthCheckRecovery(BE_HEALTH_CHECK_STR);
 470             } else {
 471                 BeEcompErrorManager.getInstance().logBeHealthCheckError(BE_HEALTH_CHECK_STR);
 472             }
 473         }
 474     }
 475
 476 }