2 * ============LICENSE_START=======================================================
4 * ================================================================================
5 * Copyright (C) 2020 AT&T Intellectual Property. All rights reserved.
6 * ================================================================================
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 * ============LICENSE_END=========================================================
20 package org.openecomp.sdc.fe.impl;
22 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_CATALOG_FACADE_MS;
23 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_ON_BOARDING;
25 import com.fasterxml.jackson.core.type.TypeReference;
26 import com.fasterxml.jackson.databind.ObjectMapper;
27 import com.google.common.annotations.VisibleForTesting;
28 import com.google.common.collect.Lists;
29 import com.google.gson.Gson;
30 import com.google.gson.GsonBuilder;
31 import com.google.gson.JsonSyntaxException;
32 import com.google.gson.reflect.TypeToken;
33 import java.io.IOException;
34 import java.lang.reflect.Type;
35 import java.util.ArrayList;
36 import java.util.Arrays;
37 import java.util.Collections;
38 import java.util.List;
41 import javax.servlet.ServletException;
43 import org.apache.commons.collections.CollectionUtils;
44 import org.apache.commons.lang3.StringUtils;
45 import org.apache.http.HttpStatus;
46 import org.eclipse.jetty.client.HttpClient;
47 import org.eclipse.jetty.util.ssl.SslContextFactory;
48 import org.onap.config.api.JettySSLUtils;
49 import org.openecomp.sdc.common.api.Constants;
50 import org.openecomp.sdc.common.api.HealthCheckInfo;
51 import org.openecomp.sdc.common.api.HealthCheckWrapper;
52 import org.openecomp.sdc.common.config.EcompErrorEnum;
53 import org.openecomp.sdc.common.http.client.api.HttpRequest;
54 import org.openecomp.sdc.common.http.client.api.HttpResponse;
55 import org.openecomp.sdc.common.http.config.ClientCertificate;
56 import org.openecomp.sdc.common.http.config.HttpClientConfig;
57 import org.openecomp.sdc.common.http.config.Timeouts;
58 import org.openecomp.sdc.common.impl.ExternalConfiguration;
59 import org.openecomp.sdc.common.log.elements.ErrorLogOptionalData;
60 import org.openecomp.sdc.common.log.elements.LogFieldsMdcHandler;
61 import org.openecomp.sdc.common.log.enums.EcompLoggerErrorCode;
62 import org.openecomp.sdc.common.log.wrappers.Logger;
63 import org.openecomp.sdc.common.util.HealthCheckUtil;
64 import org.openecomp.sdc.fe.config.Configuration;
65 import org.openecomp.sdc.fe.config.FeEcompErrorManager;
67 public class HealthCheckScheduledTask implements Runnable {
69 private static final Logger healthLogger = Logger.getLogger("asdc.fe.healthcheck");
70 private static final Logger log = Logger.getLogger(HealthCheckScheduledTask.class.getName());
71 private static final String LOG_PARTNER_NAME = "SDC.FE";
72 private static final String LOG_TARGET_ENTITY_BE = "SDC.BE";
73 private static final String LOG_TARGET_ENTITY_CONFIG = "SDC.FE.Configuration";
74 private static final String LOG_TARGET_SERVICE_NAME_OB = "getOnboardingConfig";
75 private static final String LOG_TARGET_SERVICE_NAME_FACADE = "getCatalogFacadeConfig";
76 private static final String LOG_SERVICE_NAME = "/rest/healthCheck";
77 private static final String URL = "%s://%s:%s/sdc2/rest/healthCheck";
78 private static final HealthCheckUtil healthCheckUtil = new HealthCheckUtil();
79 private static final String DEBUG_CONTEXT = "HEALTH_FE";
80 private static final String EXTERNAL_HC_URL = "%s://%s:%s%s";
81 private static LogFieldsMdcHandler mdcFieldsHandler = new LogFieldsMdcHandler();
82 private static String ONBOARDING_HC_URL;
83 private static String CATALOG_FACADE_MS_HC_URL;
84 private final List<String> healthCheckFeComponents = Arrays.asList(HC_COMPONENT_ON_BOARDING, HC_COMPONENT_CATALOG_FACADE_MS);
85 private final HealthCheckService service;
87 HealthCheckScheduledTask(HealthCheckService service) {
88 this.service = service;
91 static String getOnboardingHcUrl() {
92 return ONBOARDING_HC_URL;
95 static String getCatalogFacadeMsHcUrl() {
96 return CATALOG_FACADE_MS_HC_URL;
101 mdcFieldsHandler.addInfoForErrorAndDebugLogging(LOG_PARTNER_NAME);
102 healthLogger.trace("Executing FE Health Check Task - Start");
103 HealthCheckService.HealthStatus currentHealth = checkHealth();
104 int currentHealthStatus = currentHealth.getStatusCode();
105 healthLogger.trace("Executing FE Health Check Task - Status = {}", currentHealthStatus);
106 // In case health status was changed, issue alarm/recovery
107 if (currentHealthStatus != service.getLastHealthStatus().getStatusCode()) {
108 log.trace("FE Health State Changed to {}. Issuing alarm / recovery alarm...", currentHealthStatus);
109 logFeAlarm(currentHealthStatus);
111 // Anyway, update latest response
112 service.setLastHealthStatus(currentHealth);
115 private List<HealthCheckInfo> addHostedComponentsFeHealthCheck(String baseComponent, boolean requestedByBE) {
116 String healthCheckUrl = getExternalComponentHcUrl(baseComponent);
117 String serviceName = getExternalComponentHcUri(baseComponent);
118 ErrorLogOptionalData errorLogOptionalData = ErrorLogOptionalData.newBuilder().targetEntity(baseComponent).targetServiceName(serviceName)
120 StringBuilder description = new StringBuilder("");
121 int connectTimeoutMs = 3000;
122 int readTimeoutMs = service.getConfig().getHealthCheckSocketTimeoutInMs(5000);
123 if (healthCheckUrl != null) {
124 ObjectMapper mapper = new ObjectMapper();
126 HttpClientConfig clientConfig = new HttpClientConfig(new Timeouts(connectTimeoutMs, readTimeoutMs), getHttpClientCertificate());
128 HttpResponse<String> response = HttpRequest.get(healthCheckUrl, clientConfig);
129 int beStatus = response.getStatusCode();
130 if (beStatus == HttpStatus.SC_OK || beStatus == HttpStatus.SC_INTERNAL_SERVER_ERROR) {
131 String beJsonResponse = response.getResponse();
132 return convertResponse(beJsonResponse, mapper, baseComponent, description, beStatus);
134 description.append("Response code: " + beStatus);
135 log.trace("{} Health Check Response code: {}", baseComponent, beStatus);
137 } catch (Exception e) {
138 log.error(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR, serviceName, errorLogOptionalData, baseComponent + " unexpected response ", e);
139 description.append(baseComponent + " Unexpected response: " + e.getMessage());
142 description.append(baseComponent + " health check Configuration is missing");
144 String compName = requestedByBE ? Constants.HC_COMPONENT_FE : baseComponent;
145 return Collections.singletonList(new HealthCheckInfo(compName, HealthCheckInfo.HealthCheckStatus.DOWN, null, description.toString()));
148 private ClientCertificate getHttpClientCertificate() {
149 ClientCertificate clientCertificate = new ClientCertificate();
150 boolean certificateInfoConfigured = false;
151 if (StringUtils.isNotBlank(JettySSLUtils.getSSLConfig().getKeystorePath())) {
152 clientCertificate.setKeyStore(JettySSLUtils.getSSLConfig().getKeystorePath());
153 if (StringUtils.isNotBlank(JettySSLUtils.getSSLConfig().getKeystorePass())) {
154 clientCertificate.setKeyStorePassword(JettySSLUtils.getSSLConfig().getKeystorePass(), false);
156 certificateInfoConfigured = true;
158 if (StringUtils.isNotBlank(JettySSLUtils.getSSLConfig().getTruststorePath())) {
159 clientCertificate.setTrustStore(JettySSLUtils.getSSLConfig().getTruststorePath());
160 if (StringUtils.isNotBlank(JettySSLUtils.getSSLConfig().getTruststorePass())) {
161 clientCertificate.setTrustStorePassword(JettySSLUtils.getSSLConfig().getTruststorePass());
163 certificateInfoConfigured = true;
165 return certificateInfoConfigured ? clientCertificate: null;
168 private String getExternalComponentHcUri(String baseComponent) {
169 String healthCheckUri = null;
170 switch (baseComponent) {
171 case HC_COMPONENT_ON_BOARDING:
172 healthCheckUri = service.getConfig().getOnboarding().getHealthCheckUriFe();
174 case HC_COMPONENT_CATALOG_FACADE_MS:
175 healthCheckUri = service.getConfig().getCatalogFacadeMs().getHealthCheckUri();
178 log.debug("Unsupported base component {}", baseComponent);
181 return healthCheckUri;
185 String getExternalComponentHcUrl(String baseComponent) {
186 String healthCheckUrl = null;
187 switch (baseComponent) {
188 case HC_COMPONENT_ON_BOARDING:
189 healthCheckUrl = getOnboardingHealthCheckUrl();
191 case HC_COMPONENT_CATALOG_FACADE_MS:
192 healthCheckUrl = getCatalogFacadeHealthCheckUrl();
195 log.debug("Unsupported base component {}", baseComponent);
198 return healthCheckUrl;
201 private void logFeAlarm(int lastFeStatus) {
202 switch (lastFeStatus) {
204 FeEcompErrorManager.getInstance().processEcompError(DEBUG_CONTEXT, EcompErrorEnum.FeHealthCheckRecovery, "FE Health Recovered");
205 FeEcompErrorManager.getInstance().logFeHealthCheckRecovery("FE Health Recovered");
208 FeEcompErrorManager.getInstance()
209 .processEcompError(DEBUG_CONTEXT, EcompErrorEnum.FeHealthCheckError, "Connection with ASDC-BE is probably down");
210 FeEcompErrorManager.getInstance().logFeHealthCheckError("Connection with ASDC-BE is probably down");
217 private HealthCheckService.HealthStatus checkHealth() {
218 Gson gson = new GsonBuilder().setPrettyPrinting().create();
219 Configuration config = service.getConfig();
220 HealthCheckWrapper feAggHealthCheck;
221 boolean aggregateFeStatus = false;
222 String redirectedUrl = String.format(URL, config.getBeProtocol(), config.getBeHost(),
223 Constants.HTTPS.equals(config.getBeProtocol()) ? config.getBeSslPort() : config.getBeHttpPort());
224 int connectTimeoutMs = 3000;
225 int readTimeoutMs = config.getHealthCheckSocketTimeoutInMs(5000);
226 ErrorLogOptionalData errorLogOptionalData = ErrorLogOptionalData.newBuilder().targetEntity(LOG_TARGET_ENTITY_BE)
227 .targetServiceName(LOG_SERVICE_NAME).build();
229 HttpClientConfig clientConfig = new HttpClientConfig(new Timeouts(connectTimeoutMs, readTimeoutMs), getHttpClientCertificate());
230 HttpResponse<String> response = HttpRequest.get(redirectedUrl, clientConfig);
231 log.debug("HC call to BE - status code is {}", response.getStatusCode());
232 String beJsonResponse = response.getResponse();
233 feAggHealthCheck = getFeHealthCheckInfos(gson, beJsonResponse);
234 if (response.getStatusCode() != HttpStatus.SC_INTERNAL_SERVER_ERROR) {
235 aggregateFeStatus = healthCheckUtil.getAggregateStatus(feAggHealthCheck.getComponentsInfo(), getExcludedComponentList());
237 //Getting aggregate FE status
238 return new HealthCheckService.HealthStatus(aggregateFeStatus ? HttpStatus.SC_OK : HttpStatus.SC_INTERNAL_SERVER_ERROR,
239 gson.toJson(feAggHealthCheck));
240 } catch (Exception e) {
241 log.debug("Health Check error when trying to connect to BE or external FE. Error: {}", e);
242 log.error(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR, LOG_SERVICE_NAME, errorLogOptionalData,
243 "Health Check error when trying to connect to BE or external FE.", e.getMessage());
244 FeEcompErrorManager.getInstance()
245 .processEcompError(DEBUG_CONTEXT, EcompErrorEnum.FeHealthCheckGeneralError, "Unexpected FE Health check error");
246 FeEcompErrorManager.getInstance().logFeHealthCheckGeneralError("Unexpected FE Health check error");
247 return new HealthCheckService.HealthStatus(HttpStatus.SC_INTERNAL_SERVER_ERROR, gson.toJson(getBeDownCheckInfos()));
252 List<String> getExcludedComponentList() {
253 List<String> excludedComponentList = Lists.newArrayList(service.getConfig().getHealthStatusExclude());
254 if (isCatalogFacadeMsExcluded()) {
255 if (log.isInfoEnabled()) {
256 log.info(HC_COMPONENT_CATALOG_FACADE_MS + " has been added to the Healthcheck exclude list");
258 excludedComponentList.add(HC_COMPONENT_CATALOG_FACADE_MS);
260 return excludedComponentList;
263 private boolean isCatalogFacadeMsExcluded() {
264 //CATALOG_FACADE_MS is excluded if it is not configured
265 return service.getConfig().getCatalogFacadeMs() == null || StringUtils.isEmpty(service.getConfig().getCatalogFacadeMs().getPath());
268 private HealthCheckWrapper getFeHealthCheckInfos(Gson gson, String responseString) {
269 Type wrapperType = new TypeToken<HealthCheckWrapper>() {
271 HealthCheckWrapper healthCheckWrapper = gson.fromJson(responseString, wrapperType);
272 String description = "OK";
273 healthCheckWrapper.getComponentsInfo().add(
274 new HealthCheckInfo(Constants.HC_COMPONENT_FE, HealthCheckInfo.HealthCheckStatus.UP, ExternalConfiguration.getAppVersion(), description));
275 //add FE hosted components
276 for (String component : healthCheckFeComponents) {
277 buildHealthCheckListForComponent(component, healthCheckWrapper);
279 return healthCheckWrapper;
282 private void buildHealthCheckListForComponent(String component, HealthCheckWrapper healthCheckWrapper) {
283 HealthCheckInfo componentHCInfoFromBE = getComponentHcFromList(component, healthCheckWrapper.getComponentsInfo());
284 List<HealthCheckInfo> componentHCInfoList = addHostedComponentsFeHealthCheck(component, componentHCInfoFromBE != null);
285 HealthCheckInfo calculateStatusFor;
286 if (componentHCInfoFromBE != null) {
287 if (log.isDebugEnabled()) {
288 log.debug("{} component healthcheck info has been received from the BE and from the component itself", component);
290 //update the subcomponents's HC if exist and recalculate the component status according to the subcomponets HC
291 calculateStatusFor = updateSubComponentsInfoOfBeHc(componentHCInfoFromBE, componentHCInfoList);
293 //this component is not in the BE HC response, need to add it and calculate the aggregated status
294 if (log.isDebugEnabled()) {
295 log.debug("{} component healthcheck info has been received from the component itself, it is not monitored by the BE", component);
297 //we assume that response from components which HC is not requested by BE have only one entry in the responded list
298 calculateStatusFor = componentHCInfoList.get(0);
299 healthCheckWrapper.getComponentsInfo().add(calculateStatusFor);
301 calculateAggregatedStatus(calculateStatusFor);
305 HealthCheckInfo updateSubComponentsInfoOfBeHc(HealthCheckInfo componentHCInfoFromBE, List<HealthCheckInfo> componentHcReceivedByFE) {
306 if (!CollectionUtils.isEmpty(componentHcReceivedByFE)) {
307 //this component HC is received from BE, just need to calculate the status for that
308 if (componentHCInfoFromBE.getComponentsInfo() == null) {
309 componentHCInfoFromBE.setComponentsInfo(new ArrayList<>());
311 componentHCInfoFromBE.getComponentsInfo().addAll(componentHcReceivedByFE);
313 return componentHCInfoFromBE;
316 private HealthCheckInfo getComponentHcFromList(String component, List<HealthCheckInfo> hcList) {
317 return hcList.stream().filter(c -> c.getHealthCheckComponent().equals(component)).findFirst().orElse(null);
320 private void calculateAggregatedStatus(HealthCheckInfo baseComponentHCInfo) {
321 if (!CollectionUtils.isEmpty(baseComponentHCInfo.getComponentsInfo())) {
322 boolean status = healthCheckUtil.getAggregateStatus(baseComponentHCInfo.getComponentsInfo(), getExcludedComponentList());
323 baseComponentHCInfo.setHealthCheckStatus(status ? HealthCheckInfo.HealthCheckStatus.UP : HealthCheckInfo.HealthCheckStatus.DOWN);
324 String componentsDesc = healthCheckUtil.getAggregateDescription(baseComponentHCInfo.getComponentsInfo());
325 if (!StringUtils.isEmpty(componentsDesc)) { //aggregated description contains all the internal components desc
326 baseComponentHCInfo.setDescription(componentsDesc);
331 private HealthCheckWrapper getBeDownCheckInfos() {
332 List<HealthCheckInfo> healthCheckInfos = new ArrayList<>();
334 .add(new HealthCheckInfo(Constants.HC_COMPONENT_FE, HealthCheckInfo.HealthCheckStatus.UP, ExternalConfiguration.getAppVersion(), "OK"));
335 healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_BE, HealthCheckInfo.HealthCheckStatus.DOWN, null, null));
336 healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_JANUSGRAPH, HealthCheckInfo.HealthCheckStatus.UNKNOWN, null, null));
337 healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_CASSANDRA, HealthCheckInfo.HealthCheckStatus.UNKNOWN, null, null));
338 healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckInfo.HealthCheckStatus.UNKNOWN, null, null));
339 healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_ON_BOARDING, HealthCheckInfo.HealthCheckStatus.UNKNOWN, null, null));
340 healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_CATALOG_FACADE_MS, HealthCheckInfo.HealthCheckStatus.UNKNOWN, null, null));
341 return new HealthCheckWrapper(healthCheckInfos, "UNKNOWN", "UNKNOWN");
344 String buildHealthCheckUrl(String protocol, String host, Integer port, String uri) {
345 return String.format(EXTERNAL_HC_URL, protocol, host, port, uri);
348 private String getOnboardingHealthCheckUrl() {
349 Configuration.OnboardingConfig onboardingConfig = service.getConfig().getOnboarding();
350 ErrorLogOptionalData errorLogOptionalData = ErrorLogOptionalData.newBuilder().targetEntity(LOG_TARGET_ENTITY_CONFIG)
351 .targetServiceName(LOG_TARGET_SERVICE_NAME_OB).build();
352 if (StringUtils.isEmpty(ONBOARDING_HC_URL)) {
353 if (onboardingConfig != null) {
354 ONBOARDING_HC_URL = buildHealthCheckUrl(onboardingConfig.getProtocolFe(), onboardingConfig.getHostFe(), onboardingConfig.getPortFe(),
355 onboardingConfig.getHealthCheckUriFe());
357 log.error(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR, LOG_SERVICE_NAME, errorLogOptionalData,
358 "Onboarding health check configuration is missing.");
361 return ONBOARDING_HC_URL;
364 private String getCatalogFacadeHealthCheckUrl() {
365 Configuration.CatalogFacadeMsConfig catalogFacadeMsConfig = service.getConfig().getCatalogFacadeMs();
366 ErrorLogOptionalData errorLogOptionalData = ErrorLogOptionalData.newBuilder().targetEntity(LOG_TARGET_ENTITY_CONFIG)
367 .targetServiceName(LOG_TARGET_SERVICE_NAME_FACADE).build();
368 if (StringUtils.isEmpty(CATALOG_FACADE_MS_HC_URL)) {
369 if (catalogFacadeMsConfig != null) {
370 CATALOG_FACADE_MS_HC_URL = buildHealthCheckUrl(catalogFacadeMsConfig.getProtocol(), catalogFacadeMsConfig.getHost(),
371 catalogFacadeMsConfig.getPort(), catalogFacadeMsConfig.getHealthCheckUri());
373 log.error(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR, LOG_SERVICE_NAME, errorLogOptionalData,
374 "Catalog Facade MS health check configuration is missing.");
377 return CATALOG_FACADE_MS_HC_URL;
380 private List<HealthCheckInfo> convertResponse(String beJsonResponse, ObjectMapper mapper, String baseComponent, StringBuilder description,
382 ErrorLogOptionalData errorLogOptionalData = ErrorLogOptionalData.newBuilder().targetEntity(baseComponent).targetServiceName(LOG_SERVICE_NAME)
385 Map<String, Object> healthCheckMap = mapper.readValue(beJsonResponse, new TypeReference<Map<String, Object>>() {
387 if (healthCheckMap.containsKey("componentsInfo")) {
388 return mapper.convertValue(healthCheckMap.get("componentsInfo"), new TypeReference<List<HealthCheckInfo>>() {
391 description.append("Internal components are missing");
393 } catch (JsonSyntaxException | IOException e) {
394 log.error(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR, LOG_SERVICE_NAME, errorLogOptionalData,
395 baseComponent + " Unexpected response body ", e);
396 description.append(baseComponent).append("Unexpected response body. Response code: ").append(beStatus);
398 return new ArrayList<>();