Catalog alignment
[sdc.git] / catalog-fe / src / main / java / org / openecomp / sdc / fe / impl / HealthCheckScheduledTask.java
1 /*-
2  * ============LICENSE_START=======================================================
3  * SDC
4  * ================================================================================
5  * Copyright (C) 2020 AT&T Intellectual Property. All rights reserved.
6  * ================================================================================
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  *      http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  * ============LICENSE_END=========================================================
19  */
20
21 package org.openecomp.sdc.fe.impl;
22
23 import com.fasterxml.jackson.core.type.TypeReference;
24 import com.fasterxml.jackson.databind.ObjectMapper;
25 import com.google.common.annotations.VisibleForTesting;
26 import com.google.common.collect.Lists;
27 import com.google.gson.Gson;
28 import com.google.gson.GsonBuilder;
29 import com.google.gson.JsonSyntaxException;
30 import com.google.gson.reflect.TypeToken;
31 import org.apache.commons.collections.CollectionUtils;
32 import org.apache.commons.lang3.StringUtils;
33 import org.apache.http.HttpStatus;
34 import org.openecomp.sdc.common.api.Constants;
35 import org.openecomp.sdc.common.api.HealthCheckInfo;
36 import org.openecomp.sdc.common.api.HealthCheckWrapper;
37 import org.openecomp.sdc.common.config.EcompErrorEnum;
38 import org.openecomp.sdc.common.http.client.api.HttpRequest;
39 import org.openecomp.sdc.common.http.client.api.HttpResponse;
40 import org.openecomp.sdc.common.http.config.HttpClientConfig;
41 import org.openecomp.sdc.common.http.config.Timeouts;
42 import org.openecomp.sdc.common.impl.ExternalConfiguration;
43 import org.openecomp.sdc.common.log.elements.ErrorLogOptionalData;
44 import org.openecomp.sdc.common.log.elements.LogFieldsMdcHandler;
45 import org.openecomp.sdc.common.log.enums.EcompLoggerErrorCode;
46 import org.openecomp.sdc.common.log.wrappers.Logger;
47 import org.openecomp.sdc.common.util.HealthCheckUtil;
48 import org.openecomp.sdc.fe.config.Configuration;
49 import org.openecomp.sdc.fe.config.FeEcompErrorManager;
50
51 import java.io.IOException;
52 import java.lang.reflect.Type;
53 import java.util.ArrayList;
54 import java.util.Arrays;
55 import java.util.Collections;
56 import java.util.List;
57 import java.util.Map;
58
59 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_CATALOG_FACADE_MS;
60 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_DCAE;
61 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_ON_BOARDING;
62
63 public class HealthCheckScheduledTask implements Runnable {
64     private static final Logger healthLogger = Logger.getLogger("asdc.fe.healthcheck");
65     private static final Logger log = Logger.getLogger(HealthCheckScheduledTask.class.getName());
66     private static final String LOG_PARTNER_NAME = "SDC.FE";
67     private static final String LOG_TARGET_ENTITY_BE = "SDC.BE";
68     private static final String LOG_TARGET_ENTITY_CONFIG = "SDC.FE.Configuration";
69     private static final String LOG_TARGET_SERVICE_NAME_OB = "getOnboardingConfig";
70     private static final String LOG_TARGET_SERVICE_NAME_DCAE = "getDCAEConfig";
71     private static final String LOG_TARGET_SERVICE_NAME_FACADE = "getCatalogFacadeConfig";
72     private static final String LOG_SERVICE_NAME = "/rest/healthCheck";
73     private static LogFieldsMdcHandler mdcFieldsHandler = new LogFieldsMdcHandler();
74
75     private static final String URL = "%s://%s:%s/sdc2/rest/healthCheck";
76
77     private final List<String> healthCheckFeComponents =
78             Arrays.asList(HC_COMPONENT_ON_BOARDING, HC_COMPONENT_DCAE, HC_COMPONENT_CATALOG_FACADE_MS);
79     private static final HealthCheckUtil healthCheckUtil = new HealthCheckUtil();
80     private static final String DEBUG_CONTEXT = "HEALTH_FE";
81     private static final String EXTERNAL_HC_URL = "%s://%s:%s%s";
82     private static String ONBOARDING_HC_URL;
83     private static String DCAE_HC_URL;
84     private static String CATALOG_FACADE_MS_HC_URL;
85
86     private final HealthCheckService service;
87
88     HealthCheckScheduledTask(HealthCheckService service) {
89         this.service = service;
90     }
91
92     static String getOnboardingHcUrl() {
93         return ONBOARDING_HC_URL;
94     }
95
96     static String getDcaeHcUrl() {
97         return DCAE_HC_URL;
98     }
99
100     static String getCatalogFacadeMsHcUrl() {
101         return CATALOG_FACADE_MS_HC_URL;
102     }
103
104
105     @Override
106     public void run() {
107         mdcFieldsHandler.addInfoForErrorAndDebugLogging(LOG_PARTNER_NAME);
108         healthLogger.trace("Executing FE Health Check Task - Start");
109         HealthCheckService.HealthStatus currentHealth = checkHealth();
110         int currentHealthStatus = currentHealth.getStatusCode();
111         healthLogger.trace("Executing FE Health Check Task - Status = {}", currentHealthStatus);
112
113         // In case health status was changed, issue alarm/recovery
114         if (currentHealthStatus != service.getLastHealthStatus().getStatusCode()) {
115             log.trace("FE Health State Changed to {}. Issuing alarm / recovery alarm...", currentHealthStatus);
116             logFeAlarm(currentHealthStatus);
117         }
118         // Anyway, update latest response
119         service.setLastHealthStatus(currentHealth);
120     }
121
122     private List<HealthCheckInfo> addHostedComponentsFeHealthCheck(String baseComponent, boolean requestedByBE) {
123         String healthCheckUrl = getExternalComponentHcUrl(baseComponent);
124         String serviceName = getExternalComponentHcUri(baseComponent);
125         ErrorLogOptionalData errorLogOptionalData = ErrorLogOptionalData.newBuilder().targetEntity(baseComponent)
126                 .targetServiceName(serviceName).build();
127
128         StringBuilder description = new StringBuilder("");
129         int connectTimeoutMs = 3000;
130         int readTimeoutMs = service.getConfig().getHealthCheckSocketTimeoutInMs(5000);
131
132         if (healthCheckUrl != null) {
133             ObjectMapper mapper = new ObjectMapper();
134             try {
135                 HttpResponse<String> response = HttpRequest.get(healthCheckUrl, new HttpClientConfig(new Timeouts(connectTimeoutMs, readTimeoutMs)));
136                 int beStatus = response.getStatusCode();
137                 if (beStatus == HttpStatus.SC_OK || beStatus == HttpStatus.SC_INTERNAL_SERVER_ERROR) {
138                     String beJsonResponse = response.getResponse();
139                     return convertResponse(beJsonResponse, mapper, baseComponent, description, beStatus);
140                 } else {
141                     description.append("Response code: " + beStatus);
142                     log.trace("{} Health Check Response code: {}", baseComponent, beStatus);
143                 }
144             } catch (Exception e) {
145                 log.error(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR, serviceName, errorLogOptionalData, baseComponent + " unexpected response ", e);
146                 description.append(baseComponent + " Unexpected response: " + e.getMessage());
147             }
148         } else {
149             description.append(baseComponent + " health check Configuration is missing");
150         }
151
152         String  compName = requestedByBE ? Constants.HC_COMPONENT_FE : baseComponent;
153         return Collections.singletonList(new HealthCheckInfo(
154                 compName,
155                 HealthCheckInfo.HealthCheckStatus.DOWN,
156                 null,
157                 description.toString()));
158     }
159
160     private String getExternalComponentHcUri(String baseComponent) {
161         String healthCheckUri = null;
162         switch (baseComponent) {
163             case HC_COMPONENT_ON_BOARDING:
164                 healthCheckUri = service.getConfig().getOnboarding().getHealthCheckUriFe();
165                 break;
166             case HC_COMPONENT_DCAE:
167                 healthCheckUri = service.getConfig().getDcae().getHealthCheckUri();
168                 break;
169             case HC_COMPONENT_CATALOG_FACADE_MS:
170                 healthCheckUri = service.getConfig().getCatalogFacadeMs().getHealthCheckUri();
171                 break;
172             default:
173                 log.debug("Unsupported base component {}", baseComponent);
174                 break;
175         }
176         return healthCheckUri;
177     }
178
179
180     @VisibleForTesting
181     String getExternalComponentHcUrl(String baseComponent) {
182         String healthCheckUrl = null;
183         switch (baseComponent) {
184             case HC_COMPONENT_ON_BOARDING:
185                 healthCheckUrl = getOnboardingHealthCheckUrl();
186                 break;
187             case HC_COMPONENT_DCAE:
188                 healthCheckUrl = getDcaeHealthCheckUrl();
189                 break;
190             case HC_COMPONENT_CATALOG_FACADE_MS:
191                 healthCheckUrl = getCatalogFacadeHealthCheckUrl();
192                 break;
193             default:
194                 log.debug("Unsupported base component {}", baseComponent);
195                 break;
196         }
197         return healthCheckUrl;
198     }
199
200     private void logFeAlarm(int lastFeStatus) {
201         switch (lastFeStatus) {
202             case 200:
203                 FeEcompErrorManager.getInstance().processEcompError(DEBUG_CONTEXT, EcompErrorEnum.FeHealthCheckRecovery, "FE Health Recovered");
204                 FeEcompErrorManager.getInstance().logFeHealthCheckRecovery("FE Health Recovered");
205                 break;
206             case 500:
207                 FeEcompErrorManager.getInstance().processEcompError(DEBUG_CONTEXT, EcompErrorEnum.FeHealthCheckError, "Connection with ASDC-BE is probably down");
208                 FeEcompErrorManager.getInstance().logFeHealthCheckError("Connection with ASDC-BE is probably down");
209                 break;
210             default:
211                 break;
212         }
213     }
214
215     private HealthCheckService.HealthStatus checkHealth() {
216         Gson gson = new GsonBuilder().setPrettyPrinting().create();
217         Configuration config = service.getConfig();
218
219         HealthCheckWrapper feAggHealthCheck;
220         boolean aggregateFeStatus = false;
221         String redirectedUrl = String.format(URL, config.getBeProtocol(), config.getBeHost(),
222                 Constants.HTTPS.equals(config.getBeProtocol()) ? config.getBeSslPort() : config.getBeHttpPort());
223         int connectTimeoutMs = 3000;
224         int readTimeoutMs = config.getHealthCheckSocketTimeoutInMs(5000);
225         ErrorLogOptionalData errorLogOptionalData = ErrorLogOptionalData.newBuilder().targetEntity(LOG_TARGET_ENTITY_BE)
226                 .targetServiceName(LOG_SERVICE_NAME).build();
227
228         try {
229             HttpResponse<String> response = HttpRequest.get(redirectedUrl, new HttpClientConfig(new Timeouts(connectTimeoutMs, readTimeoutMs)));
230             log.debug("HC call to BE - status code is {}", response.getStatusCode());
231             String beJsonResponse = response.getResponse();
232             feAggHealthCheck = getFeHealthCheckInfos(gson, beJsonResponse);
233             if (response.getStatusCode() != HttpStatus.SC_INTERNAL_SERVER_ERROR) {
234                 aggregateFeStatus = healthCheckUtil.getAggregateStatus(feAggHealthCheck.getComponentsInfo(), getExcludedComponentList());
235             }
236             //Getting aggregate FE status
237             return new HealthCheckService.HealthStatus(aggregateFeStatus ? HttpStatus.SC_OK : HttpStatus.SC_INTERNAL_SERVER_ERROR, gson.toJson(feAggHealthCheck));
238
239         }
240         catch (Exception e) {
241             log.debug("Health Check error when trying to connect to BE or external FE. Error: {}", e);
242             log.error(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR, LOG_SERVICE_NAME, errorLogOptionalData,
243                     "Health Check error when trying to connect to BE or external FE.", e.getMessage());
244             FeEcompErrorManager.getInstance().processEcompError(DEBUG_CONTEXT,EcompErrorEnum.FeHealthCheckGeneralError, "Unexpected FE Health check error");
245             FeEcompErrorManager.getInstance().logFeHealthCheckGeneralError("Unexpected FE Health check error");
246             return new HealthCheckService.HealthStatus(HttpStatus.SC_INTERNAL_SERVER_ERROR, gson.toJson(getBeDownCheckInfos()));
247         }
248     }
249
250     @VisibleForTesting
251     List<String> getExcludedComponentList() {
252         List <String> excludedComponentList = Lists.newArrayList(service.getConfig().getHealthStatusExclude());
253         if (isCatalogFacadeMsExcluded()) {
254             if (log.isInfoEnabled()) {
255                 log.info(HC_COMPONENT_CATALOG_FACADE_MS + " has been added to the Healthcheck exclude list");
256             }
257             excludedComponentList.add(HC_COMPONENT_CATALOG_FACADE_MS);
258         }
259         return excludedComponentList;
260     }
261
262     private boolean isCatalogFacadeMsExcluded() {
263         //CATALOG_FACADE_MS is excluded if it is not configured
264         return service.getConfig().getCatalogFacadeMs() == null || StringUtils.isEmpty(service.getConfig().getCatalogFacadeMs().getPath());
265     }
266
267     private HealthCheckWrapper getFeHealthCheckInfos(Gson gson, String responseString) {
268         Type wrapperType = new TypeToken<HealthCheckWrapper>() {
269         }.getType();
270         HealthCheckWrapper healthCheckWrapper = gson.fromJson(responseString, wrapperType);
271         String description = "OK";
272         healthCheckWrapper.getComponentsInfo()
273                 .add(new HealthCheckInfo(Constants.HC_COMPONENT_FE, HealthCheckInfo.HealthCheckStatus.UP, ExternalConfiguration.getAppVersion(), description));
274
275         //add FE hosted components
276         for (String component : healthCheckFeComponents) {
277             buildHealthCheckListForComponent(component, healthCheckWrapper);
278         }
279         return healthCheckWrapper;
280     }
281
282     private void buildHealthCheckListForComponent(String component, HealthCheckWrapper healthCheckWrapper) {
283
284         HealthCheckInfo componentHCInfoFromBE = getComponentHcFromList(component, healthCheckWrapper.getComponentsInfo());
285         List<HealthCheckInfo> componentHCInfoList = addHostedComponentsFeHealthCheck(component, componentHCInfoFromBE != null);
286         HealthCheckInfo calculateStatusFor;
287         if (componentHCInfoFromBE != null) {
288             if (log.isDebugEnabled()) {
289                 log.debug("{} component healthcheck info has been received from the BE and from the component itself", component);
290             }
291             //update the subcomponents's HC if exist and recalculate the component status according to the subcomponets HC
292             calculateStatusFor = updateSubComponentsInfoOfBeHc(componentHCInfoFromBE, componentHCInfoList);
293         }
294         else {
295
296             //this component is not in the BE HC response, need to add it and calculate the aggregated status
297             if (log.isDebugEnabled()) {
298                 log.debug("{} component healthcheck info has been received from the component itself, it is not monitored by the BE", component);
299             }
300             //we assume that response from components which HC is not requested by BE have only one entry in the responded list
301             calculateStatusFor = componentHCInfoList.get(0);
302             healthCheckWrapper.getComponentsInfo()
303                         .add(calculateStatusFor);
304
305         }
306         calculateAggregatedStatus(calculateStatusFor);
307
308     }
309
310     @VisibleForTesting
311     HealthCheckInfo updateSubComponentsInfoOfBeHc(HealthCheckInfo componentHCInfoFromBE, List<HealthCheckInfo> componentHcReceivedByFE) {
312         if (!CollectionUtils.isEmpty(componentHcReceivedByFE)) {
313             //this component HC is received from BE, just need to calculate the status for that
314             if (componentHCInfoFromBE.getComponentsInfo() == null) {
315                 componentHCInfoFromBE.setComponentsInfo(new ArrayList<>());
316             }
317             componentHCInfoFromBE.getComponentsInfo().addAll(componentHcReceivedByFE);
318         }
319         return componentHCInfoFromBE;
320     }
321
322     private HealthCheckInfo getComponentHcFromList(String component, List<HealthCheckInfo>  hcList) {
323         return hcList.stream().filter(c -> c.getHealthCheckComponent().equals(component)).findFirst().orElse(null);
324     }
325
326     private void calculateAggregatedStatus(HealthCheckInfo baseComponentHCInfo) {
327         if (!CollectionUtils.isEmpty(baseComponentHCInfo.getComponentsInfo())) {
328             boolean status = healthCheckUtil.getAggregateStatus(baseComponentHCInfo.getComponentsInfo(), getExcludedComponentList());
329             baseComponentHCInfo.setHealthCheckStatus(status ?
330                     HealthCheckInfo.HealthCheckStatus.UP : HealthCheckInfo.HealthCheckStatus.DOWN);
331
332             String componentsDesc = healthCheckUtil.getAggregateDescription(baseComponentHCInfo.getComponentsInfo());
333             if (!StringUtils.isEmpty(componentsDesc)) { //aggregated description contains all the internal components desc
334                 baseComponentHCInfo.setDescription(componentsDesc);
335             }
336         }
337     }
338
339     private HealthCheckWrapper getBeDownCheckInfos() {
340         List<HealthCheckInfo> healthCheckInfos = new ArrayList<>();
341         healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_FE, HealthCheckInfo.HealthCheckStatus.UP,
342                 ExternalConfiguration.getAppVersion(), "OK"));
343         healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_BE, HealthCheckInfo.HealthCheckStatus.DOWN, null, null));
344         healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_JANUSGRAPH, HealthCheckInfo.HealthCheckStatus.UNKNOWN, null, null));
345         healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_CASSANDRA, HealthCheckInfo.HealthCheckStatus.UNKNOWN, null, null));
346         healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckInfo.HealthCheckStatus.UNKNOWN, null, null));
347         healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_ON_BOARDING, HealthCheckInfo.HealthCheckStatus.UNKNOWN, null, null));
348         healthCheckInfos.add(new HealthCheckInfo(Constants.HC_COMPONENT_DCAE, HealthCheckInfo.HealthCheckStatus.UNKNOWN, null, null));
349         healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_CATALOG_FACADE_MS, HealthCheckInfo.HealthCheckStatus.UNKNOWN, null, null));
350         return new HealthCheckWrapper(healthCheckInfos, "UNKNOWN", "UNKNOWN");
351     }
352
353     String buildHealthCheckUrl(String protocol, String host, Integer port, String uri) {
354         return String.format(EXTERNAL_HC_URL, protocol, host, port, uri);
355     }
356
357     private String getOnboardingHealthCheckUrl() {
358         Configuration.OnboardingConfig onboardingConfig = service.getConfig().getOnboarding();
359         ErrorLogOptionalData errorLogOptionalData = ErrorLogOptionalData.newBuilder().targetEntity(LOG_TARGET_ENTITY_CONFIG)
360                 .targetServiceName(LOG_TARGET_SERVICE_NAME_OB).build();
361
362         if (StringUtils.isEmpty(ONBOARDING_HC_URL)) {
363             if (onboardingConfig != null) {
364                 ONBOARDING_HC_URL = buildHealthCheckUrl(
365                         onboardingConfig.getProtocolFe(), onboardingConfig.getHostFe(),
366                         onboardingConfig.getPortFe(), onboardingConfig.getHealthCheckUriFe());
367             }
368             else {
369                 log.error(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR, LOG_SERVICE_NAME, errorLogOptionalData,
370                         "Onboarding health check configuration is missing.");
371             }
372         }
373         return ONBOARDING_HC_URL;
374     }
375
376     private String getDcaeHealthCheckUrl() {
377         Configuration.DcaeConfig dcaeConfig = service.getConfig().getDcae();
378         ErrorLogOptionalData errorLogOptionalData = ErrorLogOptionalData.newBuilder().targetEntity(LOG_TARGET_ENTITY_CONFIG)
379                 .targetServiceName(LOG_TARGET_SERVICE_NAME_DCAE).build();
380
381         if (StringUtils.isEmpty(DCAE_HC_URL)) {
382             if (dcaeConfig != null) {
383                 DCAE_HC_URL = buildHealthCheckUrl(
384                         dcaeConfig.getProtocol(), dcaeConfig.getHost(),
385                         dcaeConfig.getPort(), dcaeConfig.getHealthCheckUri());
386             }
387             else {
388                 log.error(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR, LOG_SERVICE_NAME, errorLogOptionalData,
389                         "DCAE health check configuration is missing.");
390             }
391         }
392         return DCAE_HC_URL;
393     }
394
395     private String getCatalogFacadeHealthCheckUrl() {
396         Configuration.CatalogFacadeMsConfig catalogFacadeMsConfig = service.getConfig().getCatalogFacadeMs();
397         ErrorLogOptionalData errorLogOptionalData = ErrorLogOptionalData.newBuilder().targetEntity(LOG_TARGET_ENTITY_CONFIG)
398                 .targetServiceName(LOG_TARGET_SERVICE_NAME_FACADE).build();
399
400         if (StringUtils.isEmpty(CATALOG_FACADE_MS_HC_URL)) {
401             if (catalogFacadeMsConfig != null) {
402                 CATALOG_FACADE_MS_HC_URL = buildHealthCheckUrl(
403                         catalogFacadeMsConfig.getProtocol(), catalogFacadeMsConfig.getHost(),
404                         catalogFacadeMsConfig.getPort(), catalogFacadeMsConfig.getHealthCheckUri());
405             }
406             else {
407                 log.error(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR, LOG_SERVICE_NAME, errorLogOptionalData,
408                         "Catalog Facade MS health check configuration is missing.");
409             }
410         }
411         return CATALOG_FACADE_MS_HC_URL;
412     }
413
414
415     private List<HealthCheckInfo> convertResponse(String beJsonResponse, ObjectMapper mapper, String baseComponent, StringBuilder description, int beStatus) {
416         ErrorLogOptionalData errorLogOptionalData = ErrorLogOptionalData.newBuilder().targetEntity(baseComponent)
417                 .targetServiceName(LOG_SERVICE_NAME).build();
418
419         try {
420             Map<String, Object> healthCheckMap = mapper.readValue(beJsonResponse, new TypeReference<Map<String, Object>>() {
421             });
422             if (healthCheckMap.containsKey("componentsInfo")) {
423                 return mapper.convertValue(healthCheckMap.get("componentsInfo"), new TypeReference<List<HealthCheckInfo>>() {
424                 });
425             } else {
426                 description.append("Internal components are missing");
427             }
428         } catch (JsonSyntaxException | IOException e) {
429             log.error(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR, LOG_SERVICE_NAME, errorLogOptionalData,
430                     baseComponent + " Unexpected response body ", e);
431             description.append(baseComponent)
432                     .append("Unexpected response body. Response code: ")
433                     .append(beStatus);
434         }
435         return new ArrayList<>();
436     }
437 }