catalog-be/src/main/java/org/openecomp/sdc/be/components/impl/HealthCheckBusinessLogic.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.be.components.impl;
  22
  23 import java.io.IOException;
  24 import java.util.ArrayList;
  25 import java.util.HashMap;
  26 import java.util.List;
  27 import java.util.Map;
  28 import java.util.Map.Entry;
  29 import java.util.concurrent.Executors;
  30 import java.util.concurrent.ScheduledExecutorService;
  31 import java.util.concurrent.ScheduledFuture;
  32 import java.util.concurrent.ThreadFactory;
  33 import java.util.concurrent.TimeUnit;
  34 import java.util.stream.Collectors;
  35
  36 import javax.annotation.PostConstruct;
  37 import javax.annotation.PreDestroy;
  38 import javax.annotation.Resource;
  39
  40 import org.apache.commons.lang3.tuple.ImmutablePair;
  41 import org.apache.commons.lang3.tuple.Pair;
  42 import org.apache.http.HttpEntity;
  43 import org.apache.http.HttpStatus;
  44 import org.apache.http.client.config.RequestConfig;
  45 import org.apache.http.client.methods.CloseableHttpResponse;
  46 import org.apache.http.client.methods.HttpGet;
  47 import org.apache.http.impl.client.CloseableHttpClient;
  48 import org.apache.http.impl.client.HttpClientBuilder;
  49 import org.apache.http.util.EntityUtils;
  50 import org.openecomp.sdc.be.components.distribution.engine.DistributionEngineClusterHealth;
  51 import org.openecomp.sdc.be.components.distribution.engine.UebHealthCheckCall;
  52 import org.openecomp.sdc.be.config.BeEcompErrorManager;
  53 import org.openecomp.sdc.be.config.Configuration;
  54 import org.openecomp.sdc.be.config.ConfigurationManager;
  55 import org.openecomp.sdc.be.dao.api.IEsHealthCheckDao;
  56 import org.openecomp.sdc.be.dao.titan.TitanGenericDao;
  57 import org.openecomp.sdc.be.switchover.detector.SwitchoverDetector;
  58 import org.openecomp.sdc.common.api.Constants;
  59 import org.openecomp.sdc.common.api.HealthCheckInfo;
  60 import org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus;
  61 import org.openecomp.sdc.common.impl.ExternalConfiguration;
  62 import org.openecomp.sdc.common.util.HealthCheckUtil;
  63 import org.slf4j.Logger;
  64 import org.slf4j.LoggerFactory;
  65 import org.springframework.beans.factory.annotation.Autowired;
  66 import org.springframework.stereotype.Component;
  67
  68 import com.fasterxml.jackson.core.type.TypeReference;
  69 import com.fasterxml.jackson.databind.ObjectMapper;
  70
  71 @Component("healthCheckBusinessLogic")
  72 public class HealthCheckBusinessLogic {
  73
  74         protected static String BE_HEALTH_LOG_CONTEXT = "be.healthcheck";
  75
  76         private static Logger healthLogger = LoggerFactory.getLogger(BE_HEALTH_LOG_CONTEXT);
  77
  78         private static final String BE_HEALTH_CHECK_STR = "beHealthCheck";
  79         private static final String COMPONENT_CHANGED_MESSAGE = "BE Component %s state changed from %s to %s";
  80
  81         @Resource
  82         private TitanGenericDao titanGenericDao;
  83
  84         @Resource
  85         private IEsHealthCheckDao esHealthCheckDao;
  86
  87         @Resource
  88         private DistributionEngineClusterHealth distributionEngineClusterHealth;
  89
  90         @Resource
  91         private CassandraHealthCheck cassandraHealthCheck;
  92
  93         @Autowired
  94         private SwitchoverDetector switchoverDetector;
  95
  96         private static Logger log = LoggerFactory.getLogger(HealthCheckBusinessLogic.class.getName());
  97
  98         private volatile List<HealthCheckInfo> prevBeHealthCheckInfos = null;
  99
 100         public HealthCheckBusinessLogic() {
 101
 102         }
 103
 104         private ScheduledFuture<?> scheduledFuture = null;
 105
 106         ScheduledExecutorService healthCheckScheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
 107                 @Override
 108                 public Thread newThread(Runnable r) {
 109                         return new Thread(r, "BE-Health-Check-Task");
 110                 }
 111         });
 112
 113         HealthCheckScheduledTask healthCheckScheduledTask = null;
 114
 115         @PostConstruct
 116         public void init() {
 117
 118                 prevBeHealthCheckInfos = getBeHealthCheckInfos();
 119
 120                 log.debug("After initializing prevBeHealthCheckInfos: {}", prevBeHealthCheckInfos);
 121
 122                 healthCheckScheduledTask = new HealthCheckScheduledTask();
 123
 124                 if (this.scheduledFuture == null) {
 125                         this.scheduledFuture = this.healthCheckScheduler.scheduleAtFixedRate(healthCheckScheduledTask, 0, 3, TimeUnit.SECONDS);
 126                 }
 127
 128         }
 129
 130         public boolean isDistributionEngineUp() {
 131
 132                 HealthCheckInfo healthCheckInfo = distributionEngineClusterHealth.getHealthCheckInfo();
 133                 if (healthCheckInfo.getHealthCheckStatus().equals(HealthCheckStatus.DOWN)) {
 134                         return false;
 135                 }
 136                 return true;
 137         }
 138
 139         public Pair<Boolean, List<HealthCheckInfo>> getBeHealthCheckInfosStatus() {
 140
 141                 return new ImmutablePair<Boolean, List<HealthCheckInfo>>(getAggregateBeStatus(prevBeHealthCheckInfos), prevBeHealthCheckInfos);
 142
 143         }
 144
 145         private Boolean getAggregateBeStatus(List<HealthCheckInfo> beHealthCheckInfos) {
 146
 147                 Boolean status = true;
 148
 149                 for (HealthCheckInfo healthCheckInfo : beHealthCheckInfos) {
 150                         if (healthCheckInfo.getHealthCheckStatus().equals(HealthCheckStatus.DOWN) && !healthCheckInfo.getHealthCheckComponent().equals(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE)) {
 151                                 status = false;
 152                                 break;
 153                         }
 154                 }
 155                 return status;
 156         }
 157
 158
 159         private List<HealthCheckInfo> getBeHealthCheckInfos() {
 160
 161                 log.trace("In getBeHealthCheckInfos");
 162
 163                 List<HealthCheckInfo> healthCheckInfos = new ArrayList<HealthCheckInfo>();
 164
 165                 // BE
 166                 getBeHealthCheck(healthCheckInfos);
 167
 168                 // Titan
 169                 getTitanHealthCheck(healthCheckInfos);
 170
 171                 // Distribution Engine
 172                 getDistributionEngineCheck(healthCheckInfos);
 173
 174                 //Cassandra
 175                 getCassandraHealthCheck(healthCheckInfos);
 176
 177                 // Amdocs
 178                 getAmdocsHealthCheck(healthCheckInfos);
 179
 180                 //DCAE
 181                 getDcaeHealthCheck(healthCheckInfos);
 182
 183                 return healthCheckInfos;
 184         }
 185
 186         private List<HealthCheckInfo> getBeHealthCheck(List<HealthCheckInfo> healthCheckInfos) {
 187                 String appVersion = ExternalConfiguration.getAppVersion();
 188                 String description = "OK";
 189                 healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_BE, HealthCheckStatus.UP, appVersion, description));
 190                 return healthCheckInfos;
 191         }
 192
 193         //Removed from aggregate HC - TDP 293490
 194         /*      private List<HealthCheckInfo> getEsHealthCheck(List<HealthCheckInfo> healthCheckInfos) {
 195
 196                 // ES health check and version
 197                 HealthCheckStatus healthCheckStatus;
 198                 String description;
 199
 200                 try {
 201                         healthCheckStatus = esHealthCheckDao.getClusterHealthStatus();
 202                 } catch (Exception e) {
 203                         healthCheckStatus = HealthCheckStatus.DOWN;
 204                         description = "ES cluster error: " + e.getMessage();
 205                         healthCheckInfos.add(new HealthCheckInfo(HealthCheckComponent.ES, healthCheckStatus, null, description));
 206                         return healthCheckInfos;
 207                 }
 208                 if (healthCheckStatus.equals(HealthCheckStatus.DOWN)) {
 209                         description = "ES cluster is down";
 210                 } else {
 211                         description = "OK";
 212                 }
 213                 healthCheckInfos.add(new HealthCheckInfo(HealthCheckComponent.ES, healthCheckStatus, null, description));
 214                 return healthCheckInfos;
 215         }
 216          */
 217         public List<HealthCheckInfo> getTitanHealthCheck(List<HealthCheckInfo> healthCheckInfos) {
 218                 // Titan health check and version
 219                 String description;
 220                 boolean isTitanUp;
 221
 222                 try {
 223                         isTitanUp = titanGenericDao.isGraphOpen();
 224                 } catch (Exception e) {
 225                         description = "Titan error: " + e.getMessage();
 226                         healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_TITAN, HealthCheckStatus.DOWN, null, description));
 227                         return healthCheckInfos;
 228                 }
 229                 if (isTitanUp) {
 230                         description = "OK";
 231                         healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_TITAN, HealthCheckStatus.UP, null, description));
 232                 } else {
 233                         description = "Titan graph is down";
 234                         healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_TITAN, HealthCheckStatus.DOWN, null, description));
 235                 }
 236                 return healthCheckInfos;
 237         }
 238
 239         private List<HealthCheckInfo> getCassandraHealthCheck(List<HealthCheckInfo> healthCheckInfos)  {
 240
 241                 String description;
 242                 boolean isCassandraUp;
 243
 244                 try {
 245                         isCassandraUp = cassandraHealthCheck.getCassandraStatus();
 246                 } catch (Exception e) {
 247                         isCassandraUp = false;
 248                         description = "Cassandra error: " + e.getMessage();
 249                 }
 250                 if (isCassandraUp) {
 251                         description = "OK";
 252                         healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_CASSANDRA, HealthCheckStatus.UP, null, description));
 253                 } else {
 254                         description = "Cassandra is down";
 255                         healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_CASSANDRA, HealthCheckStatus.DOWN, null, description));
 256                 }
 257                 return healthCheckInfos;
 258
 259         }
 260
 261         private void getDistributionEngineCheck(List<HealthCheckInfo> healthCheckInfos) {
 262
 263                 HealthCheckInfo healthCheckInfo = distributionEngineClusterHealth.getHealthCheckInfo();
 264
 265                 healthCheckInfos.add(healthCheckInfo);
 266
 267         }
 268
 269         private List<HealthCheckInfo> getAmdocsHealthCheck(List<HealthCheckInfo> healthCheckInfos) {
 270                 HealthCheckInfo beHealthCheckInfo = getHostedComponentsBeHealthCheck(Constants.HC_COMPONENT_ON_BOARDING, buildOnBoardingHealthCheckUrl());
 271                 healthCheckInfos.add(beHealthCheckInfo);
 272                 return healthCheckInfos;
 273         }
 274
 275         private List<HealthCheckInfo> getDcaeHealthCheck(List<HealthCheckInfo> healthCheckInfos) {
 276                 HealthCheckInfo beHealthCheckInfo = getHostedComponentsBeHealthCheck(Constants.HC_COMPONENT_DCAE, buildDcaeHealthCheckUrl());
 277                 healthCheckInfos.add(beHealthCheckInfo);
 278                 return healthCheckInfos;
 279         }
 280
 281         private HealthCheckInfo getHostedComponentsBeHealthCheck(String componentName, String healthCheckUrl) {
 282                 HealthCheckStatus healthCheckStatus;
 283                 String description;
 284                 String version = null;
 285                 List<HealthCheckInfo> componentsInfo = new ArrayList<>();
 286
 287                 CloseableHttpClient httpClient = getHttpClient();
 288
 289                 if (healthCheckUrl != null) {
 290                         HttpGet httpGet = new HttpGet(healthCheckUrl);
 291                         CloseableHttpResponse beResponse;
 292                         int beStatus;
 293                         try {
 294                                 beResponse = httpClient.execute(httpGet);
 295                                 beStatus = beResponse.getStatusLine().getStatusCode();
 296
 297                                 String aggDescription = "";
 298
 299                                 if (beStatus == HttpStatus.SC_OK || beStatus == HttpStatus.SC_INTERNAL_SERVER_ERROR) {
 300                                         HttpEntity entity = beResponse.getEntity();
 301                                         String beJsonResponse = EntityUtils.toString(entity);
 302                                         log.trace("{} Health Check response: {}", componentName, beJsonResponse);
 303
 304                                         ObjectMapper mapper = new ObjectMapper();
 305                                         Map<String, Object> healthCheckMap = mapper.readValue(beJsonResponse, new TypeReference<Map<String, Object>>(){});
 306                                         version = healthCheckMap.get("sdcVersion") != null ? healthCheckMap.get("sdcVersion").toString() : null;
 307                                         if (healthCheckMap.containsKey("componentsInfo")) {
 308                                                 componentsInfo = mapper.convertValue(healthCheckMap.get("componentsInfo"), new TypeReference<List<HealthCheckInfo>>() {});
 309                                         }
 310
 311                                         if (componentsInfo.size() > 0) {
 312                                                 aggDescription = HealthCheckUtil.getAggregateDescription(componentsInfo, null);
 313                                         } else {
 314                                                 componentsInfo.add(new HealthCheckInfo(Constants.HC_COMPONENT_BE, HealthCheckStatus.DOWN, null, null));
 315                                         }
 316                                 } else {
 317                                         log.trace("{} Health Check Response code: {}", componentName, beStatus);
 318                                 }
 319
 320                                 if (beStatus != HttpStatus.SC_OK) {
 321                                         healthCheckStatus = HealthCheckStatus.DOWN;
 322                                         description = aggDescription.length() > 0
 323                                                         ? aggDescription
 324                                                         : componentName + " is Down, specific reason unknown";//No inner component returned DOWN, but the status of HC is still DOWN.
 325                                                         if (componentsInfo.size() == 0) {
 326                                                                 componentsInfo.add(new HealthCheckInfo(Constants.HC_COMPONENT_BE, HealthCheckStatus.DOWN, null, description));
 327                                                         }
 328                                 } else {
 329                                         healthCheckStatus = HealthCheckStatus.UP;
 330                                         description = "OK";
 331                                 }
 332
 333                         } catch (Exception e) {
 334                                 log.error("{} unexpected response: ", componentName, e);
 335                                 healthCheckStatus = HealthCheckStatus.DOWN;
 336                                 description = componentName + " unexpected response: " + e.getMessage();
 337                                 if (componentsInfo != null && componentsInfo.size() == 0) {
 338                                         componentsInfo.add(new HealthCheckInfo(Constants.HC_COMPONENT_BE, HealthCheckStatus.DOWN, null, description));
 339                                 }
 340                         } finally {
 341                                 if (httpClient != null) {
 342                                         try {
 343                                                 httpClient.close();
 344                                         } catch (IOException e) {
 345                                                 log.error("closing http client has failed" , e);
 346                                         }
 347                                 }
 348                         }
 349                 } else {
 350                         healthCheckStatus = HealthCheckStatus.DOWN;
 351                         description = componentName + " health check Configuration is missing";
 352                         componentsInfo.add(new HealthCheckInfo(Constants.HC_COMPONENT_BE, HealthCheckStatus.DOWN, null, description));
 353                 }
 354
 355                 return new HealthCheckInfo(componentName, healthCheckStatus, version, description, componentsInfo);
 356         }
 357
 358         private CloseableHttpClient getHttpClient() {
 359                 int timeout = 3000;
 360                 RequestConfig.Builder requestBuilder = RequestConfig.custom();
 361                 requestBuilder.setConnectTimeout(timeout).setConnectionRequestTimeout(timeout).setSocketTimeout(timeout);
 362
 363                 HttpClientBuilder builder = HttpClientBuilder.create();
 364                 builder.setDefaultRequestConfig(requestBuilder.build());
 365                 return builder.build();
 366         }
 367
 368         @PreDestroy
 369         private void destroy() {
 370
 371                 if (scheduledFuture != null) {
 372                         scheduledFuture.cancel(true);
 373                         scheduledFuture = null;
 374                 }
 375
 376                 if (healthCheckScheduler != null) {
 377                         healthCheckScheduler.shutdown();
 378                 }
 379
 380         }
 381
 382         public class HealthCheckScheduledTask implements Runnable {
 383
 384                 List<UebHealthCheckCall> healthCheckCalls = new ArrayList<>();
 385
 386                 public HealthCheckScheduledTask() {
 387
 388                 }
 389
 390                 @Override
 391                 public void run() {
 392
 393                         healthLogger.trace("Executing BE Health Check Task");
 394
 395                         List<HealthCheckInfo> currentBeHealthCheckInfos = getBeHealthCheckInfos();
 396                         boolean healthStatus = getAggregateBeStatus(currentBeHealthCheckInfos);
 397
 398                         boolean prevHealthStatus = getAggregateBeStatus(prevBeHealthCheckInfos);
 399
 400                         boolean anyStatusChanged = anyStatusChanged(currentBeHealthCheckInfos, prevBeHealthCheckInfos);
 401
 402                         if (prevHealthStatus != healthStatus || anyStatusChanged) {
 403                                 log.trace("BE Health State Changed to {}. Issuing alarm / recovery alarm...", healthStatus);
 404
 405                                 prevBeHealthCheckInfos = currentBeHealthCheckInfos;
 406                                 logAlarm(healthStatus);
 407                         }
 408
 409                 }
 410         }
 411
 412         private void logAlarm(boolean prevHealthState) {
 413                 if (prevHealthState) {
 414                         BeEcompErrorManager.getInstance().logBeHealthCheckRecovery(BE_HEALTH_CHECK_STR);
 415                 } else {
 416                         BeEcompErrorManager.getInstance().logBeHealthCheckError(BE_HEALTH_CHECK_STR);
 417                 }
 418         }
 419
 420         private void logAlarm(String componentChangedMsg) {
 421                 BeEcompErrorManager.getInstance().logBeHealthCheckRecovery(componentChangedMsg);
 422         }
 423
 424
 425         public String getSiteMode() {
 426                 return switchoverDetector.getSiteMode();
 427         }
 428
 429         public boolean anyStatusChanged(List<HealthCheckInfo> beHealthCheckInfos, List<HealthCheckInfo> prevBeHealthCheckInfos) {
 430
 431                 boolean result = false;
 432
 433                 if (beHealthCheckInfos != null && prevBeHealthCheckInfos != null) {
 434
 435                         Map<String, HealthCheckStatus> currentValues = beHealthCheckInfos.stream().collect(Collectors.toMap(p -> p.getHealthCheckComponent(), p -> p.getHealthCheckStatus()));
 436                         Map<String, HealthCheckStatus> prevValues = prevBeHealthCheckInfos.stream().collect(Collectors.toMap(p -> p.getHealthCheckComponent(), p -> p.getHealthCheckStatus()));
 437
 438                         if (currentValues != null && prevValues != null) {
 439                                 int currentSize = currentValues.size();
 440                                 int prevSize = prevValues.size();
 441
 442                                 if (currentSize != prevSize) {
 443
 444                                         result = true; //extra/missing component
 445
 446                                         Map<String, HealthCheckStatus> notPresent = null;
 447                                         if (currentValues.keySet().containsAll(prevValues.keySet())) {
 448                                                 notPresent = new HashMap<>(currentValues);
 449                                                 notPresent.keySet().removeAll(prevValues.keySet());
 450                                         } else {
 451                                                 notPresent = new HashMap<>(prevValues);
 452                                                 notPresent.keySet().removeAll(currentValues.keySet());
 453                                         }
 454
 455                                         for (String component : notPresent.keySet()) {
 456                                                 logAlarm(String.format(COMPONENT_CHANGED_MESSAGE, component, prevValues.get(component), currentValues.get(component)));
 457                                         }
 458                                         //                                      HealthCheckComponent changedComponent = notPresent.keySet().iterator().next();
 459
 460                                 } else {
 461
 462                                         for (Entry<String, HealthCheckStatus> entry : currentValues.entrySet()) {
 463                                                 String key = entry.getKey();
 464                                                 HealthCheckStatus value = entry.getValue();
 465
 466                                                 if (!prevValues.containsKey(key)) {
 467                                                         result = true; //component missing
 468                                                         logAlarm(String.format(COMPONENT_CHANGED_MESSAGE, key, prevValues.get(key), currentValues.get(key)));
 469                                                         break;
 470                                                 }
 471
 472                                                 HealthCheckStatus prevHealthCheckStatus = prevValues.get(key);
 473
 474                                                 if (value != prevHealthCheckStatus) {
 475                                                         result = true; //component status changed
 476                                                         logAlarm(String.format(COMPONENT_CHANGED_MESSAGE, key, prevValues.get(key), currentValues.get(key)));
 477                                                         break;
 478                                                 }
 479                                         }
 480                                 }
 481                         }
 482
 483                 } else if (beHealthCheckInfos == null && prevBeHealthCheckInfos == null) {
 484                         result = false;
 485                 } else {
 486                         logAlarm(String.format(COMPONENT_CHANGED_MESSAGE, "", prevBeHealthCheckInfos == null ? "null" : "true", prevBeHealthCheckInfos == null ? "true" : "null"));
 487                         result = true;
 488                 }
 489
 490                 return result;
 491         }
 492
 493         private String buildOnBoardingHealthCheckUrl() {
 494
 495                 Configuration.OnboardingConfig onboardingConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getOnboarding();
 496
 497                 if (onboardingConfig != null) {
 498                         String protocol = onboardingConfig.getProtocol();
 499                         String host = onboardingConfig.getHost();
 500                         Integer port = onboardingConfig.getPort();
 501                         String uri = onboardingConfig.getHealthCheckUri();
 502
 503                         return protocol + "://" + host + ":" + port + uri;
 504                 }
 505
 506                 log.error("onboarding health check configuration is missing.");
 507                 return null;
 508         }
 509
 510         private String buildDcaeHealthCheckUrl() {
 511
 512                 Configuration.DcaeConfig dcaeConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getDcae();
 513
 514                 if (dcaeConfig != null) {
 515                         String protocol = dcaeConfig.getProtocol();
 516                         String host = dcaeConfig.getHost();
 517                         Integer port = dcaeConfig.getPort();
 518                         String uri = dcaeConfig.getHealthCheckUri();
 519
 520                         return protocol + "://" + host + ":" + port + uri;
 521                 }
 522
 523                 log.error("dcae health check configuration is missing.");
 524                 return null;
 525         }
 526 }