Fix exception in healthcheck with http
[sdc.git] / catalog-fe / src / main / java / org / openecomp / sdc / fe / impl / HealthCheckScheduledTask.java
1 /*-
2  * ============LICENSE_START=======================================================
3  * SDC
4  * ================================================================================
5  * Copyright (C) 2020 AT&T Intellectual Property. All rights reserved.
6  * ================================================================================
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  *      http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  * ============LICENSE_END=========================================================
19  */
20 package org.openecomp.sdc.fe.impl;
21
22 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_CATALOG_FACADE_MS;
23 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_ON_BOARDING;
24
25 import com.fasterxml.jackson.core.type.TypeReference;
26 import com.fasterxml.jackson.databind.ObjectMapper;
27 import com.google.common.annotations.VisibleForTesting;
28 import com.google.common.collect.Lists;
29 import com.google.gson.Gson;
30 import com.google.gson.GsonBuilder;
31 import com.google.gson.JsonSyntaxException;
32 import com.google.gson.reflect.TypeToken;
33 import java.io.IOException;
34 import java.lang.reflect.Type;
35 import java.util.ArrayList;
36 import java.util.Arrays;
37 import java.util.Collections;
38 import java.util.List;
39 import java.util.Map;
40
41 import javax.servlet.ServletException;
42
43 import org.apache.commons.collections.CollectionUtils;
44 import org.apache.commons.lang3.StringUtils;
45 import org.apache.http.HttpStatus;
46 import org.eclipse.jetty.client.HttpClient;
47 import org.eclipse.jetty.util.ssl.SslContextFactory;
48 import org.onap.config.api.JettySSLUtils;
49 import org.openecomp.sdc.common.api.Constants;
50 import org.openecomp.sdc.common.api.HealthCheckInfo;
51 import org.openecomp.sdc.common.api.HealthCheckWrapper;
52 import org.openecomp.sdc.common.config.EcompErrorEnum;
53 import org.openecomp.sdc.common.http.client.api.HttpRequest;
54 import org.openecomp.sdc.common.http.client.api.HttpResponse;
55 import org.openecomp.sdc.common.http.config.ClientCertificate;
56 import org.openecomp.sdc.common.http.config.HttpClientConfig;
57 import org.openecomp.sdc.common.http.config.Timeouts;
58 import org.openecomp.sdc.common.impl.ExternalConfiguration;
59 import org.openecomp.sdc.common.log.elements.ErrorLogOptionalData;
60 import org.openecomp.sdc.common.log.elements.LogFieldsMdcHandler;
61 import org.openecomp.sdc.common.log.enums.EcompLoggerErrorCode;
62 import org.openecomp.sdc.common.log.wrappers.Logger;
63 import org.openecomp.sdc.common.util.HealthCheckUtil;
64 import org.openecomp.sdc.fe.config.Configuration;
65 import org.openecomp.sdc.fe.config.FeEcompErrorManager;
66
67 public class HealthCheckScheduledTask implements Runnable {
68
69     private static final Logger healthLogger = Logger.getLogger("asdc.fe.healthcheck");
70     private static final Logger log = Logger.getLogger(HealthCheckScheduledTask.class.getName());
71     private static final String LOG_PARTNER_NAME = "SDC.FE";
72     private static final String LOG_TARGET_ENTITY_BE = "SDC.BE";
73     private static final String LOG_TARGET_ENTITY_CONFIG = "SDC.FE.Configuration";
74     private static final String LOG_TARGET_SERVICE_NAME_OB = "getOnboardingConfig";
75     private static final String LOG_TARGET_SERVICE_NAME_FACADE = "getCatalogFacadeConfig";
76     private static final String LOG_SERVICE_NAME = "/rest/healthCheck";
77     private static final String URL = "%s://%s:%s/sdc2/rest/healthCheck";
78     private static final HealthCheckUtil healthCheckUtil = new HealthCheckUtil();
79     private static final String DEBUG_CONTEXT = "HEALTH_FE";
80     private static final String EXTERNAL_HC_URL = "%s://%s:%s%s";
81     private static LogFieldsMdcHandler mdcFieldsHandler = new LogFieldsMdcHandler();
82     private static String ONBOARDING_HC_URL;
83     private static String CATALOG_FACADE_MS_HC_URL;
84     private final List<String> healthCheckFeComponents = Arrays.asList(HC_COMPONENT_ON_BOARDING, HC_COMPONENT_CATALOG_FACADE_MS);
85     private final HealthCheckService service;
86
87     HealthCheckScheduledTask(HealthCheckService service) {
88         this.service = service;
89     }
90
91     static String getOnboardingHcUrl() {
92         return ONBOARDING_HC_URL;
93     }
94
95     static String getCatalogFacadeMsHcUrl() {
96         return CATALOG_FACADE_MS_HC_URL;
97     }
98
99     @Override
100     public void run() {
101         mdcFieldsHandler.addInfoForErrorAndDebugLogging(LOG_PARTNER_NAME);
102         healthLogger.trace("Executing FE Health Check Task - Start");
103         HealthCheckService.HealthStatus currentHealth = checkHealth();
104         int currentHealthStatus = currentHealth.getStatusCode();
105         healthLogger.trace("Executing FE Health Check Task - Status = {}", currentHealthStatus);
106         // In case health status was changed, issue alarm/recovery
107         if (currentHealthStatus != service.getLastHealthStatus().getStatusCode()) {
108             log.trace("FE Health State Changed to {}. Issuing alarm / recovery alarm...", currentHealthStatus);
109             logFeAlarm(currentHealthStatus);
110         }
111         // Anyway, update latest response
112         service.setLastHealthStatus(currentHealth);
113     }
114
115     private List<HealthCheckInfo> addHostedComponentsFeHealthCheck(String baseComponent, boolean requestedByBE) {
116         String healthCheckUrl = getExternalComponentHcUrl(baseComponent);
117         String serviceName = getExternalComponentHcUri(baseComponent);
118         ErrorLogOptionalData errorLogOptionalData = ErrorLogOptionalData.newBuilder().targetEntity(baseComponent).targetServiceName(serviceName)
119             .build();
120         StringBuilder description = new StringBuilder("");
121         int connectTimeoutMs = 3000;
122         int readTimeoutMs = service.getConfig().getHealthCheckSocketTimeoutInMs(5000);
123         if (healthCheckUrl != null) {
124             ObjectMapper mapper = new ObjectMapper();
125             try {
126                 HttpClientConfig clientConfig = new HttpClientConfig(new Timeouts(connectTimeoutMs, readTimeoutMs), getHttpClientCertificate());
127                 
128                 HttpResponse<String> response = HttpRequest.get(healthCheckUrl, clientConfig);
129                 int beStatus = response.getStatusCode();
130                 if (beStatus == HttpStatus.SC_OK || beStatus == HttpStatus.SC_INTERNAL_SERVER_ERROR) {
131                     String beJsonResponse = response.getResponse();
132                     return convertResponse(beJsonResponse, mapper, baseComponent, description, beStatus);
133                 } else {
134                     description.append("Response code: " + beStatus);
135                     log.trace("{} Health Check Response code: {}", baseComponent, beStatus);
136                 }
137             } catch (Exception e) {
138                 log.error(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR, serviceName, errorLogOptionalData, baseComponent + " unexpected response ", e);
139                 description.append(baseComponent + " Unexpected response: " + e.getMessage());
140             }
141         } else {
142             description.append(baseComponent + " health check Configuration is missing");
143         }
144         String compName = requestedByBE ? Constants.HC_COMPONENT_FE : baseComponent;
145         return Collections.singletonList(new HealthCheckInfo(compName, HealthCheckInfo.HealthCheckStatus.DOWN, null, description.toString()));
146     }
147     
148     private ClientCertificate getHttpClientCertificate() {
149         ClientCertificate clientCertificate = new ClientCertificate();
150         boolean certificateInfoConfigured = false;
151         if (StringUtils.isNotBlank(JettySSLUtils.getSSLConfig().getKeystorePath())) {
152             clientCertificate.setKeyStore(JettySSLUtils.getSSLConfig().getKeystorePath());
153             if (StringUtils.isNotBlank(JettySSLUtils.getSSLConfig().getKeystorePass())) {
154                 clientCertificate.setKeyStorePassword(JettySSLUtils.getSSLConfig().getKeystorePass(), false);
155             }
156             certificateInfoConfigured = true;
157         }
158         if (StringUtils.isNotBlank(JettySSLUtils.getSSLConfig().getTruststorePath())) {
159             clientCertificate.setTrustStore(JettySSLUtils.getSSLConfig().getTruststorePath());
160             if (StringUtils.isNotBlank(JettySSLUtils.getSSLConfig().getTruststorePass())) {
161                 clientCertificate.setTrustStorePassword(JettySSLUtils.getSSLConfig().getTruststorePass());
162             }
163             certificateInfoConfigured = true;
164         }
165         return certificateInfoConfigured ? clientCertificate: null;
166     }
167
168     private String getExternalComponentHcUri(String baseComponent) {
169         String healthCheckUri = null;
170         switch (baseComponent) {
171             case HC_COMPONENT_ON_BOARDING:
172                 healthCheckUri = service.getConfig().getOnboarding().getHealthCheckUriFe();
173                 break;
174             case HC_COMPONENT_CATALOG_FACADE_MS:
175                 healthCheckUri = service.getConfig().getCatalogFacadeMs().getHealthCheckUri();
176                 break;
177             default:
178                 log.debug("Unsupported base component {}", baseComponent);
179                 break;
180         }
181         return healthCheckUri;
182     }
183
184     @VisibleForTesting
185     String getExternalComponentHcUrl(String baseComponent) {
186         String healthCheckUrl = null;
187         switch (baseComponent) {
188             case HC_COMPONENT_ON_BOARDING:
189                 healthCheckUrl = getOnboardingHealthCheckUrl();
190                 break;
191             case HC_COMPONENT_CATALOG_FACADE_MS:
192                 healthCheckUrl = getCatalogFacadeHealthCheckUrl();
193                 break;
194             default:
195                 log.debug("Unsupported base component {}", baseComponent);
196                 break;
197         }
198         return healthCheckUrl;
199     }
200
201     private void logFeAlarm(int lastFeStatus) {
202         switch (lastFeStatus) {
203             case 200:
204                 FeEcompErrorManager.getInstance().processEcompError(DEBUG_CONTEXT, EcompErrorEnum.FeHealthCheckRecovery, "FE Health Recovered");
205                 FeEcompErrorManager.getInstance().logFeHealthCheckRecovery("FE Health Recovered");
206                 break;
207             case 500:
208                 FeEcompErrorManager.getInstance()
209                     .processEcompError(DEBUG_CONTEXT, EcompErrorEnum.FeHealthCheckError, "Connection with ASDC-BE is probably down");
210                 FeEcompErrorManager.getInstance().logFeHealthCheckError("Connection with ASDC-BE is probably down");
211                 break;
212             default:
213                 break;
214         }
215     }
216
217     private HealthCheckService.HealthStatus checkHealth() {
218         Gson gson = new GsonBuilder().setPrettyPrinting().create();
219         Configuration config = service.getConfig();
220         HealthCheckWrapper feAggHealthCheck;
221         boolean aggregateFeStatus = false;
222         String redirectedUrl = String.format(URL, config.getBeProtocol(), config.getBeHost(),
223             Constants.HTTPS.equals(config.getBeProtocol()) ? config.getBeSslPort() : config.getBeHttpPort());
224         int connectTimeoutMs = 3000;
225         int readTimeoutMs = config.getHealthCheckSocketTimeoutInMs(5000);
226         ErrorLogOptionalData errorLogOptionalData = ErrorLogOptionalData.newBuilder().targetEntity(LOG_TARGET_ENTITY_BE)
227             .targetServiceName(LOG_SERVICE_NAME).build();
228         try {
229             HttpClientConfig clientConfig = new HttpClientConfig(new Timeouts(connectTimeoutMs, readTimeoutMs), getHttpClientCertificate());
230             HttpResponse<String> response = HttpRequest.get(redirectedUrl, clientConfig);
231             log.debug("HC call to BE - status code is {}", response.getStatusCode());
232             String beJsonResponse = response.getResponse();
233             feAggHealthCheck = getFeHealthCheckInfos(gson, beJsonResponse);
234             if (response.getStatusCode() != HttpStatus.SC_INTERNAL_SERVER_ERROR) {
235                 aggregateFeStatus = healthCheckUtil.getAggregateStatus(feAggHealthCheck.getComponentsInfo(), getExcludedComponentList());
236             }
237             //Getting aggregate FE status
238             return new HealthCheckService.HealthStatus(aggregateFeStatus ? HttpStatus.SC_OK : HttpStatus.SC_INTERNAL_SERVER_ERROR,
239                 gson.toJson(feAggHealthCheck));
240         } catch (Exception e) {
241             log.debug("Health Check error when trying to connect to BE or external FE. Error: {}", e);
242             log.error(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR, LOG_SERVICE_NAME, errorLogOptionalData,
243                 "Health Check error when trying to connect to BE or external FE.", e.getMessage());
244             FeEcompErrorManager.getInstance()
245                 .processEcompError(DEBUG_CONTEXT, EcompErrorEnum.FeHealthCheckGeneralError, "Unexpected FE Health check error");
246             FeEcompErrorManager.getInstance().logFeHealthCheckGeneralError("Unexpected FE Health check error");
247             return new HealthCheckService.HealthStatus(HttpStatus.SC_INTERNAL_SERVER_ERROR, gson.toJson(getBeDownCheckInfos()));
248         }
249     }
250
251     @VisibleForTesting
252     List<String> getExcludedComponentList() {
253         List<String> excludedComponentList = Lists.newArrayList(service.getConfig().getHealthStatusExclude());
254         if (isCatalogFacadeMsExcluded()) {
255             if (log.isInfoEnabled()) {
256                 log.info(HC_COMPONENT_CATALOG_FACADE_MS + " has been added to the Healthcheck exclude list");
257             }
258             excludedComponentList.add(HC_COMPONENT_CATALOG_FACADE_MS);
259         }
260         return excludedComponentList;
261     }
262
263     private boolean isCatalogFacadeMsExcluded() {
264         //CATALOG_FACADE_MS is excluded if it is not configured
265         return service.getConfig().getCatalogFacadeMs() == null || StringUtils.isEmpty(service.getConfig().getCatalogFacadeMs().getPath());
266     }
267
268     private HealthCheckWrapper getFeHealthCheckInfos(Gson gson, String responseString) {
269         Type wrapperType = new TypeToken<HealthCheckWrapper>() {
270         }.getType();
271         HealthCheckWrapper healthCheckWrapper = gson.fromJson(responseString, wrapperType);
272         String description = "OK";
273         healthCheckWrapper.getComponentsInfo().add(
274             new HealthCheckInfo(Constants.HC_COMPONENT_FE, HealthCheckInfo.HealthCheckStatus.UP, ExternalConfiguration.getAppVersion(), description));
275         //add FE hosted components
276         for (String component : healthCheckFeComponents) {
277             buildHealthCheckListForComponent(component, healthCheckWrapper);
278         }
279         return healthCheckWrapper;
280     }
281
282     private void buildHealthCheckListForComponent(String component, HealthCheckWrapper healthCheckWrapper) {
283         HealthCheckInfo componentHCInfoFromBE = getComponentHcFromList(component, healthCheckWrapper.getComponentsInfo());
284         List<HealthCheckInfo> componentHCInfoList = addHostedComponentsFeHealthCheck(component, componentHCInfoFromBE != null);
285         HealthCheckInfo calculateStatusFor;
286         if (componentHCInfoFromBE != null) {
287             if (log.isDebugEnabled()) {
288                 log.debug("{} component healthcheck info has been received from the BE and from the component itself", component);
289             }
290             //update the subcomponents's HC if exist and recalculate the component status according to the subcomponets HC
291             calculateStatusFor = updateSubComponentsInfoOfBeHc(componentHCInfoFromBE, componentHCInfoList);
292         } else {
293             //this component is not in the BE HC response, need to add it and calculate the aggregated status
294             if (log.isDebugEnabled()) {
295                 log.debug("{} component healthcheck info has been received from the component itself, it is not monitored by the BE", component);
296             }
297             //we assume that response from components which HC is not requested by BE have only one entry in the responded list
298             calculateStatusFor = componentHCInfoList.get(0);
299             healthCheckWrapper.getComponentsInfo().add(calculateStatusFor);
300         }
301         calculateAggregatedStatus(calculateStatusFor);
302     }
303
304     @VisibleForTesting
305     HealthCheckInfo updateSubComponentsInfoOfBeHc(HealthCheckInfo componentHCInfoFromBE, List<HealthCheckInfo> componentHcReceivedByFE) {
306         if (!CollectionUtils.isEmpty(componentHcReceivedByFE)) {
307             //this component HC is received from BE, just need to calculate the status for that
308             if (componentHCInfoFromBE.getComponentsInfo() == null) {
309                 componentHCInfoFromBE.setComponentsInfo(new ArrayList<>());
310             }
311             componentHCInfoFromBE.getComponentsInfo().addAll(componentHcReceivedByFE);
312         }
313         return componentHCInfoFromBE;
314     }
315
316     private HealthCheckInfo getComponentHcFromList(String component, List<HealthCheckInfo> hcList) {
317         return hcList.stream().filter(c -> c.getHealthCheckComponent().equals(component)).findFirst().orElse(null);
318     }
319
320     private void calculateAggregatedStatus(HealthCheckInfo baseComponentHCInfo) {
321         if (!CollectionUtils.isEmpty(baseComponentHCInfo.getComponentsInfo())) {
322             boolean status = healthCheckUtil.getAggregateStatus(baseComponentHCInfo.getComponentsInfo(), getExcludedComponentList());
323             baseComponentHCInfo.setHealthCheckStatus(status ? HealthCheckInfo.HealthCheckStatus.UP : HealthCheckInfo.HealthCheckStatus.DOWN);
324             String componentsDesc = healthCheckUtil.getAggregateDescription(baseComponentHCInfo.getComponentsInfo());
325             if (!StringUtils.isEmpty(componentsDesc)) { //aggregated description contains all the internal components desc
326                 baseComponentHCInfo.setDescription(componentsDesc);
327             }
328         }
329     }
330
331     private HealthCheckWrapper getBeDownCheckInfos() {
332         List<HealthCheckInfo> healthCheckInfos = new ArrayList<>();
333         healthCheckInfos
334             .add(new HealthCheckInfo(Constants.HC_COMPONENT_FE, HealthCheckInfo.HealthCheckStatus.UP, ExternalConfiguration.getAppVersion(), "OK"));
335         healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_BE, HealthCheckInfo.HealthCheckStatus.DOWN, null, null));
336         healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_JANUSGRAPH, HealthCheckInfo.HealthCheckStatus.UNKNOWN, null, null));
337         healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_CASSANDRA, HealthCheckInfo.HealthCheckStatus.UNKNOWN, null, null));
338         healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckInfo.HealthCheckStatus.UNKNOWN, null, null));
339         healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_ON_BOARDING, HealthCheckInfo.HealthCheckStatus.UNKNOWN, null, null));
340         healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_CATALOG_FACADE_MS, HealthCheckInfo.HealthCheckStatus.UNKNOWN, null, null));
341         return new HealthCheckWrapper(healthCheckInfos, "UNKNOWN", "UNKNOWN");
342     }
343
344     String buildHealthCheckUrl(String protocol, String host, Integer port, String uri) {
345         return String.format(EXTERNAL_HC_URL, protocol, host, port, uri);
346     }
347
348     private String getOnboardingHealthCheckUrl() {
349         Configuration.OnboardingConfig onboardingConfig = service.getConfig().getOnboarding();
350         ErrorLogOptionalData errorLogOptionalData = ErrorLogOptionalData.newBuilder().targetEntity(LOG_TARGET_ENTITY_CONFIG)
351             .targetServiceName(LOG_TARGET_SERVICE_NAME_OB).build();
352         if (StringUtils.isEmpty(ONBOARDING_HC_URL)) {
353             if (onboardingConfig != null) {
354                 ONBOARDING_HC_URL = buildHealthCheckUrl(onboardingConfig.getProtocolFe(), onboardingConfig.getHostFe(), onboardingConfig.getPortFe(),
355                     onboardingConfig.getHealthCheckUriFe());
356             } else {
357                 log.error(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR, LOG_SERVICE_NAME, errorLogOptionalData,
358                     "Onboarding health check configuration is missing.");
359             }
360         }
361         return ONBOARDING_HC_URL;
362     }
363
364     private String getCatalogFacadeHealthCheckUrl() {
365         Configuration.CatalogFacadeMsConfig catalogFacadeMsConfig = service.getConfig().getCatalogFacadeMs();
366         ErrorLogOptionalData errorLogOptionalData = ErrorLogOptionalData.newBuilder().targetEntity(LOG_TARGET_ENTITY_CONFIG)
367             .targetServiceName(LOG_TARGET_SERVICE_NAME_FACADE).build();
368         if (StringUtils.isEmpty(CATALOG_FACADE_MS_HC_URL)) {
369             if (catalogFacadeMsConfig != null) {
370                 CATALOG_FACADE_MS_HC_URL = buildHealthCheckUrl(catalogFacadeMsConfig.getProtocol(), catalogFacadeMsConfig.getHost(),
371                     catalogFacadeMsConfig.getPort(), catalogFacadeMsConfig.getHealthCheckUri());
372             } else {
373                 log.error(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR, LOG_SERVICE_NAME, errorLogOptionalData,
374                     "Catalog Facade MS health check configuration is missing.");
375             }
376         }
377         return CATALOG_FACADE_MS_HC_URL;
378     }
379
380     private List<HealthCheckInfo> convertResponse(String beJsonResponse, ObjectMapper mapper, String baseComponent, StringBuilder description,
381                                                   int beStatus) {
382         ErrorLogOptionalData errorLogOptionalData = ErrorLogOptionalData.newBuilder().targetEntity(baseComponent).targetServiceName(LOG_SERVICE_NAME)
383             .build();
384         try {
385             Map<String, Object> healthCheckMap = mapper.readValue(beJsonResponse, new TypeReference<Map<String, Object>>() {
386             });
387             if (healthCheckMap.containsKey("componentsInfo")) {
388                 return mapper.convertValue(healthCheckMap.get("componentsInfo"), new TypeReference<List<HealthCheckInfo>>() {
389                 });
390             } else {
391                 description.append("Internal components are missing");
392             }
393         } catch (JsonSyntaxException | IOException e) {
394             log.error(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR, LOG_SERVICE_NAME, errorLogOptionalData,
395                 baseComponent + " Unexpected response body ", e);
396             description.append(baseComponent).append("Unexpected response body. Response code: ").append(beStatus);
397         }
398         return new ArrayList<>();
399     }
400 }