catalog-fe/src/main/java/org/openecomp/sdc/fe/impl/HealthCheckScheduledTask.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2020 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20 package org.openecomp.sdc.fe.impl;
  21
  22 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_CATALOG_FACADE_MS;
  23 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_ON_BOARDING;
  24
  25 import com.fasterxml.jackson.core.type.TypeReference;
  26 import com.fasterxml.jackson.databind.ObjectMapper;
  27 import com.google.common.annotations.VisibleForTesting;
  28 import com.google.common.collect.Lists;
  29 import com.google.gson.Gson;
  30 import com.google.gson.GsonBuilder;
  31 import com.google.gson.JsonSyntaxException;
  32 import com.google.gson.reflect.TypeToken;
  33 import java.io.IOException;
  34 import java.lang.reflect.Type;
  35 import java.util.ArrayList;
  36 import java.util.Arrays;
  37 import java.util.Collections;
  38 import java.util.List;
  39 import java.util.Map;
  40
  41 import javax.servlet.ServletException;
  42
  43 import org.apache.commons.collections.CollectionUtils;
  44 import org.apache.commons.lang3.StringUtils;
  45 import org.apache.http.HttpStatus;
  46 import org.eclipse.jetty.client.HttpClient;
  47 import org.eclipse.jetty.util.ssl.SslContextFactory;
  48 import org.onap.config.api.JettySSLUtils;
  49 import org.openecomp.sdc.common.api.Constants;
  50 import org.openecomp.sdc.common.api.HealthCheckInfo;
  51 import org.openecomp.sdc.common.api.HealthCheckWrapper;
  52 import org.openecomp.sdc.common.config.EcompErrorEnum;
  53 import org.openecomp.sdc.common.http.client.api.HttpRequest;
  54 import org.openecomp.sdc.common.http.client.api.HttpResponse;
  55 import org.openecomp.sdc.common.http.config.ClientCertificate;
  56 import org.openecomp.sdc.common.http.config.HttpClientConfig;
  57 import org.openecomp.sdc.common.http.config.Timeouts;
  58 import org.openecomp.sdc.common.impl.ExternalConfiguration;
  59 import org.openecomp.sdc.common.log.elements.ErrorLogOptionalData;
  60 import org.openecomp.sdc.common.log.elements.LogFieldsMdcHandler;
  61 import org.openecomp.sdc.common.log.enums.EcompLoggerErrorCode;
  62 import org.openecomp.sdc.common.log.wrappers.Logger;
  63 import org.openecomp.sdc.common.util.HealthCheckUtil;
  64 import org.openecomp.sdc.fe.config.Configuration;
  65 import org.openecomp.sdc.fe.config.FeEcompErrorManager;
  66
  67 public class HealthCheckScheduledTask implements Runnable {
  68
  69     private static final Logger healthLogger = Logger.getLogger("asdc.fe.healthcheck");
  70     private static final Logger log = Logger.getLogger(HealthCheckScheduledTask.class.getName());
  71     private static final String LOG_PARTNER_NAME = "SDC.FE";
  72     private static final String LOG_TARGET_ENTITY_BE = "SDC.BE";
  73     private static final String LOG_TARGET_ENTITY_CONFIG = "SDC.FE.Configuration";
  74     private static final String LOG_TARGET_SERVICE_NAME_OB = "getOnboardingConfig";
  75     private static final String LOG_TARGET_SERVICE_NAME_FACADE = "getCatalogFacadeConfig";
  76     private static final String LOG_SERVICE_NAME = "/rest/healthCheck";
  77     private static final String URL = "%s://%s:%s/sdc2/rest/healthCheck";
  78     private static final HealthCheckUtil healthCheckUtil = new HealthCheckUtil();
  79     private static final String DEBUG_CONTEXT = "HEALTH_FE";
  80     private static final String EXTERNAL_HC_URL = "%s://%s:%s%s";
  81     private static LogFieldsMdcHandler mdcFieldsHandler = new LogFieldsMdcHandler();
  82     private static String ONBOARDING_HC_URL;
  83     private static String CATALOG_FACADE_MS_HC_URL;
  84     private final List<String> healthCheckFeComponents = Arrays.asList(HC_COMPONENT_ON_BOARDING, HC_COMPONENT_CATALOG_FACADE_MS);
  85     private final HealthCheckService service;
  86
  87     HealthCheckScheduledTask(HealthCheckService service) {
  88         this.service = service;
  89     }
  90
  91     static String getOnboardingHcUrl() {
  92         return ONBOARDING_HC_URL;
  93     }
  94
  95     static String getCatalogFacadeMsHcUrl() {
  96         return CATALOG_FACADE_MS_HC_URL;
  97     }
  98
  99     @Override
 100     public void run() {
 101         mdcFieldsHandler.addInfoForErrorAndDebugLogging(LOG_PARTNER_NAME);
 102         healthLogger.trace("Executing FE Health Check Task - Start");
 103         HealthCheckService.HealthStatus currentHealth = checkHealth();
 104         int currentHealthStatus = currentHealth.getStatusCode();
 105         healthLogger.trace("Executing FE Health Check Task - Status = {}", currentHealthStatus);
 106         // In case health status was changed, issue alarm/recovery
 107         if (currentHealthStatus != service.getLastHealthStatus().getStatusCode()) {
 108             log.trace("FE Health State Changed to {}. Issuing alarm / recovery alarm...", currentHealthStatus);
 109             logFeAlarm(currentHealthStatus);
 110         }
 111         // Anyway, update latest response
 112         service.setLastHealthStatus(currentHealth);
 113     }
 114
 115     private List<HealthCheckInfo> addHostedComponentsFeHealthCheck(String baseComponent, boolean requestedByBE) {
 116         String healthCheckUrl = getExternalComponentHcUrl(baseComponent);
 117         String serviceName = getExternalComponentHcUri(baseComponent);
 118         ErrorLogOptionalData errorLogOptionalData = ErrorLogOptionalData.newBuilder().targetEntity(baseComponent).targetServiceName(serviceName)
 119             .build();
 120         StringBuilder description = new StringBuilder("");
 121         int connectTimeoutMs = 3000;
 122         int readTimeoutMs = service.getConfig().getHealthCheckSocketTimeoutInMs(5000);
 123         if (healthCheckUrl != null) {
 124             ObjectMapper mapper = new ObjectMapper();
 125             try {
 126                 HttpClientConfig clientConfig = new HttpClientConfig(new Timeouts(connectTimeoutMs, readTimeoutMs), getHttpClientCertificate());
 127
 128                 HttpResponse<String> response = HttpRequest.get(healthCheckUrl, clientConfig);
 129                 int beStatus = response.getStatusCode();
 130                 if (beStatus == HttpStatus.SC_OK || beStatus == HttpStatus.SC_INTERNAL_SERVER_ERROR) {
 131                     String beJsonResponse = response.getResponse();
 132                     return convertResponse(beJsonResponse, mapper, baseComponent, description, beStatus);
 133                 } else {
 134                     description.append("Response code: " + beStatus);
 135                     log.trace("{} Health Check Response code: {}", baseComponent, beStatus);
 136                 }
 137             } catch (Exception e) {
 138                 log.error(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR, serviceName, errorLogOptionalData, baseComponent + " unexpected response ", e);
 139                 description.append(baseComponent + " Unexpected response: " + e.getMessage());
 140             }
 141         } else {
 142             description.append(baseComponent + " health check Configuration is missing");
 143         }
 144         String compName = requestedByBE ? Constants.HC_COMPONENT_FE : baseComponent;
 145         return Collections.singletonList(new HealthCheckInfo(compName, HealthCheckInfo.HealthCheckStatus.DOWN, null, description.toString()));
 146     }
 147
 148     private ClientCertificate getHttpClientCertificate() {
 149         ClientCertificate clientCertificate = new ClientCertificate();
 150         clientCertificate.setKeyStore(JettySSLUtils.getSSLConfig().getKeystorePath());
 151         clientCertificate.setKeyStorePassword(JettySSLUtils.getSSLConfig().getKeystorePass(), false);
 152         clientCertificate.setTrustStore(JettySSLUtils.getSSLConfig().getTruststorePath());
 153         clientCertificate.setTrustStorePassword(JettySSLUtils.getSSLConfig().getTruststorePass());
 154         return clientCertificate;
 155     }
 156
 157     private String getExternalComponentHcUri(String baseComponent) {
 158         String healthCheckUri = null;
 159         switch (baseComponent) {
 160             case HC_COMPONENT_ON_BOARDING:
 161                 healthCheckUri = service.getConfig().getOnboarding().getHealthCheckUriFe();
 162                 break;
 163             case HC_COMPONENT_CATALOG_FACADE_MS:
 164                 healthCheckUri = service.getConfig().getCatalogFacadeMs().getHealthCheckUri();
 165                 break;
 166             default:
 167                 log.debug("Unsupported base component {}", baseComponent);
 168                 break;
 169         }
 170         return healthCheckUri;
 171     }
 172
 173     @VisibleForTesting
 174     String getExternalComponentHcUrl(String baseComponent) {
 175         String healthCheckUrl = null;
 176         switch (baseComponent) {
 177             case HC_COMPONENT_ON_BOARDING:
 178                 healthCheckUrl = getOnboardingHealthCheckUrl();
 179                 break;
 180             case HC_COMPONENT_CATALOG_FACADE_MS:
 181                 healthCheckUrl = getCatalogFacadeHealthCheckUrl();
 182                 break;
 183             default:
 184                 log.debug("Unsupported base component {}", baseComponent);
 185                 break;
 186         }
 187         return healthCheckUrl;
 188     }
 189
 190     private void logFeAlarm(int lastFeStatus) {
 191         switch (lastFeStatus) {
 192             case 200:
 193                 FeEcompErrorManager.getInstance().processEcompError(DEBUG_CONTEXT, EcompErrorEnum.FeHealthCheckRecovery, "FE Health Recovered");
 194                 FeEcompErrorManager.getInstance().logFeHealthCheckRecovery("FE Health Recovered");
 195                 break;
 196             case 500:
 197                 FeEcompErrorManager.getInstance()
 198                     .processEcompError(DEBUG_CONTEXT, EcompErrorEnum.FeHealthCheckError, "Connection with ASDC-BE is probably down");
 199                 FeEcompErrorManager.getInstance().logFeHealthCheckError("Connection with ASDC-BE is probably down");
 200                 break;
 201             default:
 202                 break;
 203         }
 204     }
 205
 206     private HealthCheckService.HealthStatus checkHealth() {
 207         Gson gson = new GsonBuilder().setPrettyPrinting().create();
 208         Configuration config = service.getConfig();
 209         HealthCheckWrapper feAggHealthCheck;
 210         boolean aggregateFeStatus = false;
 211         String redirectedUrl = String.format(URL, config.getBeProtocol(), config.getBeHost(),
 212             Constants.HTTPS.equals(config.getBeProtocol()) ? config.getBeSslPort() : config.getBeHttpPort());
 213         int connectTimeoutMs = 3000;
 214         int readTimeoutMs = config.getHealthCheckSocketTimeoutInMs(5000);
 215         ErrorLogOptionalData errorLogOptionalData = ErrorLogOptionalData.newBuilder().targetEntity(LOG_TARGET_ENTITY_BE)
 216             .targetServiceName(LOG_SERVICE_NAME).build();
 217         try {
 218             HttpClientConfig clientConfig = new HttpClientConfig(new Timeouts(connectTimeoutMs, readTimeoutMs), getHttpClientCertificate());
 219             HttpResponse<String> response = HttpRequest.get(redirectedUrl, clientConfig);
 220             log.debug("HC call to BE - status code is {}", response.getStatusCode());
 221             String beJsonResponse = response.getResponse();
 222             feAggHealthCheck = getFeHealthCheckInfos(gson, beJsonResponse);
 223             if (response.getStatusCode() != HttpStatus.SC_INTERNAL_SERVER_ERROR) {
 224                 aggregateFeStatus = healthCheckUtil.getAggregateStatus(feAggHealthCheck.getComponentsInfo(), getExcludedComponentList());
 225             }
 226             //Getting aggregate FE status
 227             return new HealthCheckService.HealthStatus(aggregateFeStatus ? HttpStatus.SC_OK : HttpStatus.SC_INTERNAL_SERVER_ERROR,
 228                 gson.toJson(feAggHealthCheck));
 229         } catch (Exception e) {
 230             log.debug("Health Check error when trying to connect to BE or external FE. Error: {}", e);
 231             log.error(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR, LOG_SERVICE_NAME, errorLogOptionalData,
 232                 "Health Check error when trying to connect to BE or external FE.", e.getMessage());
 233             FeEcompErrorManager.getInstance()
 234                 .processEcompError(DEBUG_CONTEXT, EcompErrorEnum.FeHealthCheckGeneralError, "Unexpected FE Health check error");
 235             FeEcompErrorManager.getInstance().logFeHealthCheckGeneralError("Unexpected FE Health check error");
 236             return new HealthCheckService.HealthStatus(HttpStatus.SC_INTERNAL_SERVER_ERROR, gson.toJson(getBeDownCheckInfos()));
 237         }
 238     }
 239
 240     @VisibleForTesting
 241     List<String> getExcludedComponentList() {
 242         List<String> excludedComponentList = Lists.newArrayList(service.getConfig().getHealthStatusExclude());
 243         if (isCatalogFacadeMsExcluded()) {
 244             if (log.isInfoEnabled()) {
 245                 log.info(HC_COMPONENT_CATALOG_FACADE_MS + " has been added to the Healthcheck exclude list");
 246             }
 247             excludedComponentList.add(HC_COMPONENT_CATALOG_FACADE_MS);
 248         }
 249         return excludedComponentList;
 250     }
 251
 252     private boolean isCatalogFacadeMsExcluded() {
 253         //CATALOG_FACADE_MS is excluded if it is not configured
 254         return service.getConfig().getCatalogFacadeMs() == null || StringUtils.isEmpty(service.getConfig().getCatalogFacadeMs().getPath());
 255     }
 256
 257     private HealthCheckWrapper getFeHealthCheckInfos(Gson gson, String responseString) {
 258         Type wrapperType = new TypeToken<HealthCheckWrapper>() {
 259         }.getType();
 260         HealthCheckWrapper healthCheckWrapper = gson.fromJson(responseString, wrapperType);
 261         String description = "OK";
 262         healthCheckWrapper.getComponentsInfo().add(
 263             new HealthCheckInfo(Constants.HC_COMPONENT_FE, HealthCheckInfo.HealthCheckStatus.UP, ExternalConfiguration.getAppVersion(), description));
 264         //add FE hosted components
 265         for (String component : healthCheckFeComponents) {
 266             buildHealthCheckListForComponent(component, healthCheckWrapper);
 267         }
 268         return healthCheckWrapper;
 269     }
 270
 271     private void buildHealthCheckListForComponent(String component, HealthCheckWrapper healthCheckWrapper) {
 272         HealthCheckInfo componentHCInfoFromBE = getComponentHcFromList(component, healthCheckWrapper.getComponentsInfo());
 273         List<HealthCheckInfo> componentHCInfoList = addHostedComponentsFeHealthCheck(component, componentHCInfoFromBE != null);
 274         HealthCheckInfo calculateStatusFor;
 275         if (componentHCInfoFromBE != null) {
 276             if (log.isDebugEnabled()) {
 277                 log.debug("{} component healthcheck info has been received from the BE and from the component itself", component);
 278             }
 279             //update the subcomponents's HC if exist and recalculate the component status according to the subcomponets HC
 280             calculateStatusFor = updateSubComponentsInfoOfBeHc(componentHCInfoFromBE, componentHCInfoList);
 281         } else {
 282             //this component is not in the BE HC response, need to add it and calculate the aggregated status
 283             if (log.isDebugEnabled()) {
 284                 log.debug("{} component healthcheck info has been received from the component itself, it is not monitored by the BE", component);
 285             }
 286             //we assume that response from components which HC is not requested by BE have only one entry in the responded list
 287             calculateStatusFor = componentHCInfoList.get(0);
 288             healthCheckWrapper.getComponentsInfo().add(calculateStatusFor);
 289         }
 290         calculateAggregatedStatus(calculateStatusFor);
 291     }
 292
 293     @VisibleForTesting
 294     HealthCheckInfo updateSubComponentsInfoOfBeHc(HealthCheckInfo componentHCInfoFromBE, List<HealthCheckInfo> componentHcReceivedByFE) {
 295         if (!CollectionUtils.isEmpty(componentHcReceivedByFE)) {
 296             //this component HC is received from BE, just need to calculate the status for that
 297             if (componentHCInfoFromBE.getComponentsInfo() == null) {
 298                 componentHCInfoFromBE.setComponentsInfo(new ArrayList<>());
 299             }
 300             componentHCInfoFromBE.getComponentsInfo().addAll(componentHcReceivedByFE);
 301         }
 302         return componentHCInfoFromBE;
 303     }
 304
 305     private HealthCheckInfo getComponentHcFromList(String component, List<HealthCheckInfo> hcList) {
 306         return hcList.stream().filter(c -> c.getHealthCheckComponent().equals(component)).findFirst().orElse(null);
 307     }
 308
 309     private void calculateAggregatedStatus(HealthCheckInfo baseComponentHCInfo) {
 310         if (!CollectionUtils.isEmpty(baseComponentHCInfo.getComponentsInfo())) {
 311             boolean status = healthCheckUtil.getAggregateStatus(baseComponentHCInfo.getComponentsInfo(), getExcludedComponentList());
 312             baseComponentHCInfo.setHealthCheckStatus(status ? HealthCheckInfo.HealthCheckStatus.UP : HealthCheckInfo.HealthCheckStatus.DOWN);
 313             String componentsDesc = healthCheckUtil.getAggregateDescription(baseComponentHCInfo.getComponentsInfo());
 314             if (!StringUtils.isEmpty(componentsDesc)) { //aggregated description contains all the internal components desc
 315                 baseComponentHCInfo.setDescription(componentsDesc);
 316             }
 317         }
 318     }
 319
 320     private HealthCheckWrapper getBeDownCheckInfos() {
 321         List<HealthCheckInfo> healthCheckInfos = new ArrayList<>();
 322         healthCheckInfos
 323             .add(new HealthCheckInfo(Constants.HC_COMPONENT_FE, HealthCheckInfo.HealthCheckStatus.UP, ExternalConfiguration.getAppVersion(), "OK"));
 324         healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_BE, HealthCheckInfo.HealthCheckStatus.DOWN, null, null));
 325         healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_JANUSGRAPH, HealthCheckInfo.HealthCheckStatus.UNKNOWN, null, null));
 326         healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_CASSANDRA, HealthCheckInfo.HealthCheckStatus.UNKNOWN, null, null));
 327         healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckInfo.HealthCheckStatus.UNKNOWN, null, null));
 328         healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_ON_BOARDING, HealthCheckInfo.HealthCheckStatus.UNKNOWN, null, null));
 329         healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_CATALOG_FACADE_MS, HealthCheckInfo.HealthCheckStatus.UNKNOWN, null, null));
 330         return new HealthCheckWrapper(healthCheckInfos, "UNKNOWN", "UNKNOWN");
 331     }
 332
 333     String buildHealthCheckUrl(String protocol, String host, Integer port, String uri) {
 334         return String.format(EXTERNAL_HC_URL, protocol, host, port, uri);
 335     }
 336
 337     private String getOnboardingHealthCheckUrl() {
 338         Configuration.OnboardingConfig onboardingConfig = service.getConfig().getOnboarding();
 339         ErrorLogOptionalData errorLogOptionalData = ErrorLogOptionalData.newBuilder().targetEntity(LOG_TARGET_ENTITY_CONFIG)
 340             .targetServiceName(LOG_TARGET_SERVICE_NAME_OB).build();
 341         if (StringUtils.isEmpty(ONBOARDING_HC_URL)) {
 342             if (onboardingConfig != null) {
 343                 ONBOARDING_HC_URL = buildHealthCheckUrl(onboardingConfig.getProtocolFe(), onboardingConfig.getHostFe(), onboardingConfig.getPortFe(),
 344                     onboardingConfig.getHealthCheckUriFe());
 345             } else {
 346                 log.error(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR, LOG_SERVICE_NAME, errorLogOptionalData,
 347                     "Onboarding health check configuration is missing.");
 348             }
 349         }
 350         return ONBOARDING_HC_URL;
 351     }
 352
 353     private String getCatalogFacadeHealthCheckUrl() {
 354         Configuration.CatalogFacadeMsConfig catalogFacadeMsConfig = service.getConfig().getCatalogFacadeMs();
 355         ErrorLogOptionalData errorLogOptionalData = ErrorLogOptionalData.newBuilder().targetEntity(LOG_TARGET_ENTITY_CONFIG)
 356             .targetServiceName(LOG_TARGET_SERVICE_NAME_FACADE).build();
 357         if (StringUtils.isEmpty(CATALOG_FACADE_MS_HC_URL)) {
 358             if (catalogFacadeMsConfig != null) {
 359                 CATALOG_FACADE_MS_HC_URL = buildHealthCheckUrl(catalogFacadeMsConfig.getProtocol(), catalogFacadeMsConfig.getHost(),
 360                     catalogFacadeMsConfig.getPort(), catalogFacadeMsConfig.getHealthCheckUri());
 361             } else {
 362                 log.error(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR, LOG_SERVICE_NAME, errorLogOptionalData,
 363                     "Catalog Facade MS health check configuration is missing.");
 364             }
 365         }
 366         return CATALOG_FACADE_MS_HC_URL;
 367     }
 368
 369     private List<HealthCheckInfo> convertResponse(String beJsonResponse, ObjectMapper mapper, String baseComponent, StringBuilder description,
 370                                                   int beStatus) {
 371         ErrorLogOptionalData errorLogOptionalData = ErrorLogOptionalData.newBuilder().targetEntity(baseComponent).targetServiceName(LOG_SERVICE_NAME)
 372             .build();
 373         try {
 374             Map<String, Object> healthCheckMap = mapper.readValue(beJsonResponse, new TypeReference<Map<String, Object>>() {
 375             });
 376             if (healthCheckMap.containsKey("componentsInfo")) {
 377                 return mapper.convertValue(healthCheckMap.get("componentsInfo"), new TypeReference<List<HealthCheckInfo>>() {
 378                 });
 379             } else {
 380                 description.append("Internal components are missing");
 381             }
 382         } catch (JsonSyntaxException | IOException e) {
 383             log.error(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR, LOG_SERVICE_NAME, errorLogOptionalData,
 384                 baseComponent + " Unexpected response body ", e);
 385             description.append(baseComponent).append("Unexpected response body. Response code: ").append(beStatus);
 386         }
 387         return new ArrayList<>();
 388     }
 389 }