catalog-be/src/main/java/org/openecomp/sdc/be/components/impl/HealthCheckBusinessLogic.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.be.components.impl;
  22
  23 import java.io.IOException;
  24 import java.lang.reflect.Type;
  25 import java.util.ArrayList;
  26 import java.util.HashMap;
  27 import java.util.List;
  28 import java.util.Map;
  29 import java.util.Map.Entry;
  30 import java.util.concurrent.Executors;
  31 import java.util.concurrent.ScheduledExecutorService;
  32 import java.util.concurrent.ScheduledFuture;
  33 import java.util.concurrent.ThreadFactory;
  34 import java.util.concurrent.TimeUnit;
  35 import java.util.stream.Collectors;
  36
  37 import javax.annotation.PostConstruct;
  38 import javax.annotation.PreDestroy;
  39 import javax.annotation.Resource;
  40
  41 import org.apache.http.HttpEntity;
  42 import org.apache.http.HttpStatus;
  43 import org.apache.http.client.config.RequestConfig;
  44 import org.apache.http.client.methods.CloseableHttpResponse;
  45 import org.apache.http.client.methods.HttpGet;
  46 import org.apache.http.impl.client.CloseableHttpClient;
  47 import org.apache.http.impl.client.HttpClientBuilder;
  48 import org.apache.http.util.EntityUtils;
  49 import org.openecomp.sdc.be.components.distribution.engine.DistributionEngineClusterHealth;
  50 import org.openecomp.sdc.be.components.distribution.engine.UebHealthCheckCall;
  51 import org.openecomp.sdc.be.config.BeEcompErrorManager;
  52 import org.openecomp.sdc.be.config.Configuration;
  53 import org.openecomp.sdc.be.config.ConfigurationManager;
  54 import org.openecomp.sdc.be.dao.api.IEsHealthCheckDao;
  55 import org.openecomp.sdc.be.dao.titan.TitanGenericDao;
  56 import org.openecomp.sdc.be.switchover.detector.SwitchoverDetector;
  57 import org.openecomp.sdc.common.api.HealthCheckInfo;
  58 import org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckComponent;
  59 import org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus;
  60 import org.openecomp.sdc.common.impl.ExternalConfiguration;
  61 import org.slf4j.Logger;
  62 import org.slf4j.LoggerFactory;
  63 import org.springframework.beans.factory.annotation.Autowired;
  64 import org.springframework.stereotype.Component;
  65
  66 import com.google.gson.Gson;
  67 import com.google.gson.reflect.TypeToken;
  68
  69 @Component("healthCheckBusinessLogic")
  70 public class HealthCheckBusinessLogic {
  71
  72         protected static String BE_HEALTH_LOG_CONTEXT = "be.healthcheck";
  73
  74         private static Logger healthLogger = LoggerFactory.getLogger(BE_HEALTH_LOG_CONTEXT);
  75
  76         private static final String BE_HEALTH_CHECK_STR = "beHealthCheck";
  77         private static final String COMPONENT_CHANGED_MESSAGE = "BE Component %s state changed from %s to %s";
  78
  79         @Resource
  80         private TitanGenericDao titanGenericDao;
  81
  82         @Resource
  83         private IEsHealthCheckDao esHealthCheckDao;
  84
  85         @Resource
  86         private DistributionEngineClusterHealth distributionEngineClusterHealth;
  87
  88         @Resource
  89         private CassandraHealthCheck cassandraHealthCheck;
  90
  91         @Autowired
  92         private SwitchoverDetector switchoverDetector;
  93
  94         private static Logger log = LoggerFactory.getLogger(HealthCheckBusinessLogic.class.getName());
  95
  96         private volatile List<HealthCheckInfo> prevBeHealthCheckInfos = null;
  97
  98         public HealthCheckBusinessLogic() {
  99
 100         }
 101
 102         private ScheduledFuture<?> scheduledFuture = null;
 103
 104         ScheduledExecutorService healthCheckScheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
 105                 @Override
 106                 public Thread newThread(Runnable r) {
 107                         return new Thread(r, "BE-Health-Check-Task");
 108                 }
 109         });
 110
 111         HealthCheckScheduledTask healthCheckScheduledTask = null;
 112
 113         @PostConstruct
 114         public void init() {
 115
 116                 prevBeHealthCheckInfos = getBeHealthCheckInfos();
 117
 118                 log.debug("After initializing prevBeHealthCheckInfos: {}", prevBeHealthCheckInfos);
 119
 120                 healthCheckScheduledTask = new HealthCheckScheduledTask();
 121
 122                 if (this.scheduledFuture == null) {
 123                         this.scheduledFuture = this.healthCheckScheduler.scheduleAtFixedRate(healthCheckScheduledTask, 0, 3, TimeUnit.SECONDS);
 124                 }
 125
 126         }
 127
 128         public boolean isDistributionEngineUp() {
 129
 130                 HealthCheckInfo healthCheckInfo = distributionEngineClusterHealth.getHealthCheckInfo();
 131                 if (healthCheckInfo.getHealthCheckStatus().equals(HealthCheckStatus.DOWN)) {
 132                         return false;
 133                 }
 134                 return true;
 135         }
 136
 137         public List<HealthCheckInfo> getBeHealthCheckInfosStatus() {
 138
 139                 return prevBeHealthCheckInfos;
 140
 141         }
 142
 143         private List<HealthCheckInfo> getBeHealthCheckInfos() {
 144
 145                 log.trace("In getBeHealthCheckInfos");
 146
 147                 List<HealthCheckInfo> healthCheckInfos = new ArrayList<HealthCheckInfo>();
 148
 149                 // BE
 150                 getBeHealthCheck(healthCheckInfos);
 151
 152                 /*// ES
 153                 getEsHealthCheck(healthCheckInfos);*/
 154
 155                 // Titan
 156                 getTitanHealthCheck(healthCheckInfos);
 157
 158                 // Distribution Engine
 159                 getDistributionEngineCheck(healthCheckInfos);
 160
 161                 //Cassandra
 162                 getCassandraHealthCheck(healthCheckInfos);
 163
 164                 // Amdocs
 165                 getAmdocsHealthCheck(healthCheckInfos);
 166
 167                 return healthCheckInfos;
 168         }
 169
 170         private List<HealthCheckInfo> getBeHealthCheck(List<HealthCheckInfo> healthCheckInfos) {
 171                 String appVersion = ExternalConfiguration.getAppVersion();
 172                 String description = "OK";
 173                 healthCheckInfos.add(new HealthCheckInfo(HealthCheckComponent.BE, HealthCheckStatus.UP, appVersion, description));
 174                 return healthCheckInfos;
 175         }
 176
 177         //Removed from aggregate HC - TDP 293490
 178 /*      private List<HealthCheckInfo> getEsHealthCheck(List<HealthCheckInfo> healthCheckInfos) {
 179
 180                 // ES health check and version
 181                 HealthCheckStatus healthCheckStatus;
 182                 String description;
 183
 184                 try {
 185                         healthCheckStatus = esHealthCheckDao.getClusterHealthStatus();
 186                 } catch (Exception e) {
 187                         healthCheckStatus = HealthCheckStatus.DOWN;
 188                         description = "ES cluster error: " + e.getMessage();
 189                         healthCheckInfos.add(new HealthCheckInfo(HealthCheckComponent.ES, healthCheckStatus, null, description));
 190                         return healthCheckInfos;
 191                 }
 192                 if (healthCheckStatus.equals(HealthCheckStatus.DOWN)) {
 193                         description = "ES cluster is down";
 194                 } else {
 195                         description = "OK";
 196                 }
 197                 healthCheckInfos.add(new HealthCheckInfo(HealthCheckComponent.ES, healthCheckStatus, null, description));
 198                 return healthCheckInfos;
 199         }
 200 */
 201         public List<HealthCheckInfo> getTitanHealthCheck(List<HealthCheckInfo> healthCheckInfos) {
 202                 // Titan health check and version
 203                 String description;
 204                 boolean isTitanUp;
 205
 206                 try {
 207                         isTitanUp = titanGenericDao.isGraphOpen();
 208                 } catch (Exception e) {
 209                         description = "Titan error: " + e.getMessage();
 210                         healthCheckInfos.add(new HealthCheckInfo(HealthCheckComponent.TITAN, HealthCheckStatus.DOWN, null, description));
 211                         return healthCheckInfos;
 212                 }
 213                 if (isTitanUp) {
 214                         description = "OK";
 215                         healthCheckInfos.add(new HealthCheckInfo(HealthCheckComponent.TITAN, HealthCheckStatus.UP, null, description));
 216                 } else {
 217                         description = "Titan graph is down";
 218                         healthCheckInfos.add(new HealthCheckInfo(HealthCheckComponent.TITAN, HealthCheckStatus.DOWN, null, description));
 219                 }
 220                 return healthCheckInfos;
 221         }
 222
 223         private List<HealthCheckInfo> getCassandraHealthCheck(List<HealthCheckInfo> healthCheckInfos)  {
 224
 225                 String description;
 226                 boolean isCassandraUp;
 227
 228                 try {
 229                         isCassandraUp = cassandraHealthCheck.getCassandraStatus();
 230                 } catch (Exception e) {
 231                         isCassandraUp = false;
 232                         description = "Cassandra error: " + e.getMessage();
 233                 }
 234                 if (isCassandraUp) {
 235                         description = "OK";
 236                         healthCheckInfos.add(new HealthCheckInfo(HealthCheckComponent.CASSANDRA, HealthCheckStatus.UP, null, description));
 237                 } else {
 238                         description = "Cassandra is down";
 239                         healthCheckInfos.add(new HealthCheckInfo(HealthCheckComponent.CASSANDRA, HealthCheckStatus.DOWN, null, description));
 240                 }
 241                 return healthCheckInfos;
 242
 243         }
 244
 245         private void getDistributionEngineCheck(List<HealthCheckInfo> healthCheckInfos) {
 246
 247                 HealthCheckInfo healthCheckInfo = distributionEngineClusterHealth.getHealthCheckInfo();
 248
 249                 healthCheckInfos.add(healthCheckInfo);
 250
 251         }
 252
 253         private List<HealthCheckInfo> getAmdocsHealthCheck(List<HealthCheckInfo> healthCheckInfos) {
 254                 HealthCheckStatus healthCheckStatus;
 255                 String description;
 256                 Map<String, Object> amdocsHC = null;
 257                 String version = null;
 258                 List<HealthCheckInfo> componentsInfo = null;
 259                 CloseableHttpClient httpClient = getHttpClient();
 260                 String amdocsHealtchCheckUrl = buildHealthCheckUrl();
 261                 HttpGet httpGet = new HttpGet(amdocsHealtchCheckUrl);
 262                 CloseableHttpResponse beResponse;
 263                 int beStatus;
 264                 try {
 265                         beResponse = httpClient.execute(httpGet);
 266                         beStatus = beResponse.getStatusLine().getStatusCode();
 267
 268                         HttpEntity entity = beResponse.getEntity();
 269                         String beJsonResponse = EntityUtils.toString(entity);
 270                         Gson gson = new Gson();
 271                         amdocsHC = gson.fromJson(beJsonResponse, Map.class);
 272                         version = amdocsHC.get("sdcVersion") != null ? amdocsHC.get("sdcVersion").toString() : null;
 273                         Object object = amdocsHC.get("componentsInfo");
 274                         Type listType = new TypeToken<List<HealthCheckInfo>>(){}.getType();
 275                         componentsInfo = gson.fromJson(object.toString(), listType);
 276
 277                         if (beStatus != HttpStatus.SC_OK) {
 278                                 healthCheckStatus = HealthCheckStatus.DOWN;
 279                                 StringBuilder sb = new StringBuilder();
 280                                 componentsInfo.forEach(x -> {
 281                                         if (x.getHealthCheckStatus()==HealthCheckStatus.DOWN){
 282                                                 sb.append("Component "+x.getHealthCheckComponent().name()+" is Down,");
 283                                         }
 284                                 });
 285                                 //Removing the last comma
 286                                 description = sb.length()>0
 287                                                 ? sb.substring(0, sb.length()-1)
 288                                                                 : "Onboarding is Down, specific reason unknown";//No Amdocs inner component returned DOWN, but the status of Amdocs HC is still DOWN.
 289                         } else {
 290                                 healthCheckStatus = HealthCheckStatus.UP;
 291                                 description = "OK";
 292
 293
 294                         }
 295
 296                 } catch (Exception e) {
 297                         healthCheckStatus = HealthCheckStatus.DOWN;
 298                         description = "Onboarding unexpected response: " + e.getMessage();
 299                 } finally {
 300                         if (httpClient != null) {
 301                                 try {
 302                                         httpClient.close();
 303                                 } catch (IOException e) {
 304                                         e.printStackTrace();
 305                                 }
 306                         }
 307                 }
 308
 309                 healthCheckInfos.add(new HealthCheckInfo(HealthCheckComponent.ON_BOARDING, healthCheckStatus, version, description, componentsInfo));
 310                 return healthCheckInfos;
 311         }
 312
 313         private CloseableHttpClient getHttpClient() {
 314                 int timeout = 3000;
 315                 RequestConfig.Builder requestBuilder = RequestConfig.custom();
 316                 requestBuilder.setConnectTimeout(timeout).setConnectionRequestTimeout(timeout).setSocketTimeout(timeout);
 317
 318                 HttpClientBuilder builder = HttpClientBuilder.create();
 319                 builder.setDefaultRequestConfig(requestBuilder.build());
 320                 return builder.build();
 321         }
 322
 323         @PreDestroy
 324         private void destroy() {
 325
 326                 if (scheduledFuture != null) {
 327                         scheduledFuture.cancel(true);
 328                         scheduledFuture = null;
 329                 }
 330
 331                 if (healthCheckScheduler != null) {
 332                         healthCheckScheduler.shutdown();
 333                 }
 334
 335         }
 336
 337         public class HealthCheckScheduledTask implements Runnable {
 338
 339                 List<UebHealthCheckCall> healthCheckCalls = new ArrayList<>();
 340
 341                 public HealthCheckScheduledTask() {
 342
 343                 }
 344
 345                 @Override
 346                 public void run() {
 347
 348                         healthLogger.trace("Executing BE Health Check Task");
 349
 350                         List<HealthCheckInfo> currentBeHealthCheckInfos = getBeHealthCheckInfos();
 351                         boolean healthStatus = getAggregateBeStatus(currentBeHealthCheckInfos);
 352
 353                         boolean prevHealthStatus = getAggregateBeStatus(prevBeHealthCheckInfos);
 354
 355                         boolean anyStatusChanged = anyStatusChanged(currentBeHealthCheckInfos, prevBeHealthCheckInfos);
 356
 357                         if (prevHealthStatus != healthStatus || anyStatusChanged) {
 358                                 log.trace("BE Health State Changed to {}. Issuing alarm / recovery alarm...", healthStatus);
 359
 360                                 prevBeHealthCheckInfos = currentBeHealthCheckInfos;
 361                                 logAlarm(healthStatus);
 362                         }
 363
 364                 }
 365
 366                 private boolean getAggregateBeStatus(List<HealthCheckInfo> beHealthCheckInfos) {
 367
 368                         boolean status = true;
 369
 370                         for (HealthCheckInfo healthCheckInfo : beHealthCheckInfos) {
 371                                 if (healthCheckInfo.getHealthCheckStatus().equals(HealthCheckStatus.DOWN) && healthCheckInfo.getHealthCheckComponent() != HealthCheckComponent.DE) {
 372                                         status = false;
 373                                         break;
 374                                 }
 375                         }
 376                         return status;
 377                 }
 378
 379         }
 380
 381         private void logAlarm(boolean prevHealthState) {
 382                 if (prevHealthState) {
 383                         BeEcompErrorManager.getInstance().logBeHealthCheckRecovery(BE_HEALTH_CHECK_STR);
 384                 } else {
 385                         BeEcompErrorManager.getInstance().logBeHealthCheckError(BE_HEALTH_CHECK_STR);
 386                 }
 387         }
 388
 389         private void logAlarm(String componentChangedMsg) {
 390                 BeEcompErrorManager.getInstance().logBeHealthCheckRecovery(componentChangedMsg);
 391         }
 392
 393
 394         public String getSiteMode() {
 395                 return switchoverDetector.getSiteMode();
 396         }
 397
 398         public boolean anyStatusChanged(List<HealthCheckInfo> beHealthCheckInfos, List<HealthCheckInfo> prevBeHealthCheckInfos) {
 399
 400                 boolean result = false;
 401
 402                 if (beHealthCheckInfos != null && prevBeHealthCheckInfos != null) {
 403
 404                         Map<HealthCheckComponent, HealthCheckStatus> currentValues = beHealthCheckInfos.stream().collect(Collectors.toMap(p -> p.getHealthCheckComponent(), p -> p.getHealthCheckStatus()));
 405                         Map<HealthCheckComponent, HealthCheckStatus> prevValues = prevBeHealthCheckInfos.stream().collect(Collectors.toMap(p -> p.getHealthCheckComponent(), p -> p.getHealthCheckStatus()));
 406
 407                         if (currentValues != null && prevValues != null) {
 408                                 int currentSize = currentValues.size();
 409                                 int prevSize = prevValues.size();
 410
 411                                 if (currentSize != prevSize) {
 412
 413                                         result = true; //extra/missing component
 414
 415                                         Map<HealthCheckComponent, HealthCheckStatus> notPresent = null;
 416                                         if (currentValues.keySet().containsAll(prevValues.keySet())) {
 417                                                 notPresent = new HashMap<>(currentValues);
 418                                                 notPresent.keySet().removeAll(prevValues.keySet());
 419                                         } else {
 420                                                 notPresent = new HashMap<>(prevValues);
 421                                                 notPresent.keySet().removeAll(currentValues.keySet());
 422                                         }
 423
 424                                         for (HealthCheckComponent component : notPresent.keySet()) {
 425                                                 logAlarm(String.format(COMPONENT_CHANGED_MESSAGE, component, prevValues.get(component), currentValues.get(component)));
 426                                         }
 427                                         //                                      HealthCheckComponent changedComponent = notPresent.keySet().iterator().next();
 428
 429                                 } else {
 430
 431                                         for (Entry<HealthCheckComponent, HealthCheckStatus> entry : currentValues.entrySet()) {
 432                                                 HealthCheckComponent key = entry.getKey();
 433                                                 HealthCheckStatus value = entry.getValue();
 434
 435                                                 if (!prevValues.containsKey(key)) {
 436                                                         result = true; //component missing
 437                                                         logAlarm(String.format(COMPONENT_CHANGED_MESSAGE, key, prevValues.get(key), currentValues.get(key)));
 438                                                         break;
 439                                                 }
 440
 441                                                 HealthCheckStatus prevHealthCheckStatus = prevValues.get(key);
 442
 443                                                 if (value != prevHealthCheckStatus) {
 444                                                         result = true; //component status changed
 445                                                         logAlarm(String.format(COMPONENT_CHANGED_MESSAGE, key, prevValues.get(key), currentValues.get(key)));
 446                                                         break;
 447                                                 }
 448                                         }
 449                                 }
 450                         }
 451
 452                 } else if (beHealthCheckInfos == null && prevBeHealthCheckInfos == null) {
 453                         result = false;
 454                 } else {
 455                         logAlarm(String.format(COMPONENT_CHANGED_MESSAGE, "", prevBeHealthCheckInfos == null ? "null" : "true", prevBeHealthCheckInfos == null ? "true" : "null"));
 456                         result = true;
 457                 }
 458
 459                 return result;
 460         }
 461
 462         private String buildHealthCheckUrl() {
 463
 464                 Configuration.OnboardingConfig onboardingConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getOnboarding();
 465
 466                 String protocol = onboardingConfig.getProtocol();
 467                 String host = onboardingConfig.getHost();
 468                 Integer port = onboardingConfig.getPort();
 469                 String uri = onboardingConfig.getHealthCheckUri();
 470
 471                 return protocol + "://" + host + ":" + port + uri;
 472         }
 473 }