2 * ============LICENSE_START=======================================================
4 * ================================================================================
5 * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
6 * ================================================================================
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 * ============LICENSE_END=========================================================
20 package org.openecomp.sdc.be.components.health;
22 import static java.lang.String.format;
23 import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
24 import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
25 import static org.apache.http.HttpStatus.SC_OK;
26 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_BE;
27 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_CASSANDRA;
28 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_DMAAP_PRODUCER;
29 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_ECOMP_PORTAL;
30 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_JANUSGRAPH;
31 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_ON_BOARDING;
32 import static org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus.DOWN;
33 import static org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus.UP;
34 import static org.openecomp.sdc.common.impl.ExternalConfiguration.getAppVersion;
36 import com.fasterxml.jackson.core.type.TypeReference;
37 import com.fasterxml.jackson.databind.ObjectMapper;
38 import java.util.ArrayList;
39 import java.util.HashMap;
40 import java.util.List;
42 import java.util.Map.Entry;
43 import java.util.concurrent.ScheduledExecutorService;
44 import java.util.concurrent.ScheduledFuture;
45 import java.util.concurrent.TimeUnit;
46 import java.util.stream.Collectors;
47 import javax.annotation.PostConstruct;
48 import javax.annotation.PreDestroy;
49 import javax.annotation.Resource;
50 import org.apache.commons.lang3.tuple.ImmutablePair;
51 import org.apache.commons.lang3.tuple.Pair;
52 import org.openecomp.sdc.be.catalog.impl.DmaapProducerHealth;
53 import org.openecomp.sdc.be.components.distribution.engine.DistributionEngineClusterHealth;
54 import org.openecomp.sdc.be.components.distribution.engine.DmaapHealth;
55 import org.openecomp.sdc.be.components.impl.CassandraHealthCheck;
56 import org.openecomp.sdc.be.config.BeEcompErrorManager;
57 import org.openecomp.sdc.be.config.Configuration;
58 import org.openecomp.sdc.be.config.ConfigurationManager;
59 import org.openecomp.sdc.be.dao.janusgraph.JanusGraphGenericDao;
60 import org.openecomp.sdc.be.switchover.detector.SwitchoverDetector;
61 import org.openecomp.sdc.common.api.HealthCheckInfo;
62 import org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus;
63 import org.openecomp.sdc.common.http.client.api.HttpRequest;
64 import org.openecomp.sdc.common.http.client.api.HttpResponse;
65 import org.openecomp.sdc.common.http.config.HttpClientConfig;
66 import org.openecomp.sdc.common.http.config.Timeouts;
67 import org.openecomp.sdc.common.log.elements.LogFieldsMdcHandler;
68 import org.openecomp.sdc.common.log.wrappers.Logger;
69 import org.openecomp.sdc.common.util.HealthCheckUtil;
70 import org.springframework.beans.factory.annotation.Autowired;
71 import org.springframework.stereotype.Component;
73 @Component("healthCheckBusinessLogic")
74 public class HealthCheckBusinessLogic {
76 private static String hcUrl = "%s://%s:%s%s";
77 private static final String BE_HEALTH_CHECK_STR = "beHealthCheck";
78 private static final String LOG_PARTNER_NAME = "SDC.BE";
79 private static final String COMPONENT_CHANGED_MESSAGE = "BE Component %s state changed from %s to %s";
80 private static final Logger log = Logger.getLogger(HealthCheckBusinessLogic.class.getName());
81 private static final HealthCheckUtil healthCheckUtil = new HealthCheckUtil();
82 private final ScheduledExecutorService healthCheckScheduler = newSingleThreadScheduledExecutor(
83 (Runnable r) -> new Thread(r, "BE-Health-Check-Task"));
84 private HealthCheckScheduledTask healthCheckScheduledTask = null;
85 private static LogFieldsMdcHandler mdcFieldsHandler = new LogFieldsMdcHandler();
87 private JanusGraphGenericDao janusGraphGenericDao;
89 private DistributionEngineClusterHealth distributionEngineClusterHealth;
91 private DmaapHealth dmaapHealth;
93 private DmaapProducerHealth dmaapProducerHealth;
95 private CassandraHealthCheck cassandraHealthCheck;
97 private PortalHealthCheckBuilder portalHealthCheck;
99 private SwitchoverDetector switchoverDetector;
100 private volatile List<HealthCheckInfo> prevBeHealthCheckInfos = null;
101 private ScheduledFuture<?> scheduledFuture = null;
105 prevBeHealthCheckInfos = getBeHealthCheckInfos();
106 log.debug("After initializing prevBeHealthCheckInfos: {}", prevBeHealthCheckInfos);
107 healthCheckScheduledTask = new HealthCheckScheduledTask();
108 if (this.scheduledFuture == null) {
109 this.scheduledFuture = this.healthCheckScheduler.scheduleAtFixedRate(healthCheckScheduledTask, 0, 3, TimeUnit.SECONDS);
113 public boolean isDistributionEngineUp() {
114 HealthCheckInfo healthCheckInfo = distributionEngineClusterHealth.getHealthCheckInfo();
115 return healthCheckInfo.getHealthCheckStatus() != DOWN;
118 public Pair<Boolean, List<HealthCheckInfo>> getBeHealthCheckInfosStatus() {
119 Configuration config = ConfigurationManager.getConfigurationManager().getConfiguration();
120 return new ImmutablePair<>(healthCheckUtil.getAggregateStatus(prevBeHealthCheckInfos, config.getHealthStatusExclude()),
121 prevBeHealthCheckInfos);
124 private List<HealthCheckInfo> getBeHealthCheckInfos() {
125 log.trace("In getBeHealthCheckInfos");
126 List<HealthCheckInfo> healthCheckInfos = new ArrayList<>();
128 HealthCheckInfo info;
129 if ((info = getDmaapHealthCheck()) != null) {
130 healthCheckInfos.add(info);
133 healthCheckInfos.add(getDmaapProducerHealthCheck());
135 healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_BE, UP, getAppVersion(), "OK"));
137 healthCheckInfos.add(getJanusGraphHealthCheck());
138 // Distribution Engine
139 healthCheckInfos.add(distributionEngineClusterHealth.getHealthCheckInfo());
141 healthCheckInfos.add(getCassandraHealthCheck());
143 healthCheckInfos.add(getHostedComponentsBeHealthCheck(HC_COMPONENT_ON_BOARDING, buildOnBoardingHealthCheckUrl()));
145 healthCheckInfos.add(portalHealthCheck.getHealthCheckInfo());
146 return healthCheckInfos;
149 private HealthCheckInfo getDmaapHealthCheck() {
150 HealthCheckInfo healthCheckInfo = null;
151 if (ConfigurationManager.getConfigurationManager().getConfiguration().getDmaapConsumerConfiguration().isActive()) {
152 String appVersion = getAppVersion();
153 dmaapHealth.getHealthCheckInfo().setVersion(appVersion);
154 healthCheckInfo = dmaapHealth.getHealthCheckInfo();
156 log.debug("Dmaap health check disabled");
158 return healthCheckInfo;
161 private HealthCheckInfo getDmaapProducerHealthCheck() {
162 if (ConfigurationManager.getConfigurationManager().getConfiguration().getDmaapConsumerConfiguration().isActive()) {
163 String appVersion = getAppVersion();
164 dmaapProducerHealth.getHealthCheckInfo().setVersion(appVersion);
165 return dmaapProducerHealth.getHealthCheckInfo();
167 String description = "Dmaap health check disabled";
168 log.debug(description);
169 return new HealthCheckInfo(HC_COMPONENT_DMAAP_PRODUCER, DOWN, null, description);
173 public HealthCheckInfo getJanusGraphHealthCheck() {
174 // JanusGraph health check and version
176 boolean isJanusGraphUp;
177 HealthCheckInfo healthCheckInfo = new HealthCheckInfo(HC_COMPONENT_JANUSGRAPH, DOWN, null, null);
179 isJanusGraphUp = janusGraphGenericDao.isGraphOpen();
180 } catch (Exception e) {
181 description = "JanusGraph error: " + e.getMessage();
182 healthCheckInfo.setDescription(description);
183 log.error(description);
184 return healthCheckInfo;
186 if (isJanusGraphUp) {
188 healthCheckInfo.setDescription(description);
189 healthCheckInfo.setHealthCheckStatus(HealthCheckInfo.HealthCheckStatus.UP);
191 description = "JanusGraph graph is down";
192 healthCheckInfo.setDescription(description);
194 return healthCheckInfo;
197 private HealthCheckInfo getCassandraHealthCheck() {
199 boolean isCassandraUp = false;
200 HealthCheckInfo healthCheckInfo = new HealthCheckInfo(HC_COMPONENT_CASSANDRA, DOWN, null, null);
202 isCassandraUp = cassandraHealthCheck.getCassandraStatus();
203 } catch (Exception e) {
204 description = "Cassandra error: " + e.getMessage();
205 log.error(description, e);
209 healthCheckInfo.setHealthCheckStatus(HealthCheckStatus.UP);
210 healthCheckInfo.setDescription(description);
212 description = "Cassandra is down";
213 healthCheckInfo.setDescription(description);
215 return healthCheckInfo;
218 private HealthCheckInfo getHostedComponentsBeHealthCheck(String componentName, String healthCheckUrl) {
219 HealthCheckStatus healthCheckStatus;
221 String version = null;
222 List<HealthCheckInfo> componentsInfo = new ArrayList<>();
223 final int timeout = 3000;
224 if (healthCheckUrl != null) {
226 HttpResponse<String> httpResponse = HttpRequest.get(healthCheckUrl, new HttpClientConfig(new Timeouts(timeout, timeout)));
227 int statusCode = httpResponse.getStatusCode();
228 String aggDescription = "";
229 if ((statusCode == SC_OK || statusCode == SC_INTERNAL_SERVER_ERROR) && !componentName.equals(HC_COMPONENT_ECOMP_PORTAL)) {
230 String response = httpResponse.getResponse();
231 log.trace("{} Health Check response: {}", componentName, response);
232 ObjectMapper mapper = new ObjectMapper();
233 Map<String, Object> healthCheckMap = mapper.readValue(response, new TypeReference<Map<String, Object>>() {
235 version = getVersion(healthCheckMap);
236 if (healthCheckMap.containsKey("componentsInfo")) {
237 componentsInfo = mapper.convertValue(healthCheckMap.get("componentsInfo"), new TypeReference<List<HealthCheckInfo>>() {
240 aggDescription = getAggDescription(componentsInfo, aggDescription);
242 log.trace("{} Health Check Response code: {}", componentName, statusCode);
244 if (statusCode != SC_OK) {
245 healthCheckStatus = DOWN;
246 description = getDescription(componentName, aggDescription);
247 setDescriptionToObject(description, componentsInfo);
249 healthCheckStatus = UP;
252 } catch (Exception e) {
253 log.error("{} unexpected response: ", componentName, e);
254 healthCheckStatus = DOWN;
255 description = componentName + " unexpected response: " + e.getMessage();
256 addToHealthCheckInfoObject(description, componentsInfo);
259 healthCheckStatus = DOWN;
260 description = componentName + " health check Configuration is missing";
261 componentsInfo.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, description));
263 return new HealthCheckInfo(componentName, healthCheckStatus, version, description, componentsInfo);
266 private void addToHealthCheckInfoObject(String description, List<HealthCheckInfo> componentsInfo) {
267 if (componentsInfo != null && componentsInfo.isEmpty()) {
268 componentsInfo.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, description));
272 private void setDescriptionToObject(String description, List<HealthCheckInfo> componentsInfo) {
273 if (componentsInfo.isEmpty()) {
274 componentsInfo.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, description));
278 private String getDescription(String componentName, String aggDescription) {
280 description = aggDescription.length() > 0 ? aggDescription
281 : componentName + " is Down, specific reason unknown";//No inner component returned DOWN, but the status of HC is still DOWN.
285 private String getVersion(Map<String, Object> healthCheckMap) {
286 return healthCheckMap.get("sdcVersion") != null ? healthCheckMap.get("sdcVersion").toString() : null;
289 private String getAggDescription(List<HealthCheckInfo> componentsInfo, String aggDescription) {
290 if (!componentsInfo.isEmpty()) {
291 aggDescription = healthCheckUtil.getAggregateDescription(componentsInfo);
293 componentsInfo.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, null));
295 return aggDescription;
299 protected void destroy() {
300 if (scheduledFuture != null) {
301 scheduledFuture.cancel(true);
302 scheduledFuture = null;
304 if (healthCheckScheduler != null) {
305 healthCheckScheduler.shutdown();
309 private void logAlarm(String componentChangedMsg) {
310 BeEcompErrorManager.getInstance().logBeHealthCheckRecovery(componentChangedMsg);
313 public String getSiteMode() {
314 return switchoverDetector.getSiteMode();
317 public boolean anyStatusChanged(List<HealthCheckInfo> beHealthCheckInfos, List<HealthCheckInfo> prevBeHealthCheckInfos) {
318 boolean result = false;
319 if (beHealthCheckInfos != null && prevBeHealthCheckInfos != null) {
320 Map<String, HealthCheckStatus> currentValues = beHealthCheckInfos.stream()
321 .collect(Collectors.toMap(HealthCheckInfo::getHealthCheckComponent, HealthCheckInfo::getHealthCheckStatus));
322 Map<String, HealthCheckStatus> prevValues = prevBeHealthCheckInfos.stream()
323 .collect(Collectors.toMap(HealthCheckInfo::getHealthCheckComponent, HealthCheckInfo::getHealthCheckStatus));
324 if (currentValues != null && prevValues != null) {
325 int currentSize = currentValues.size();
326 int prevSize = prevValues.size();
327 if (currentSize != prevSize) {
328 result = true; //extra/missing component
329 updateHealthCheckStatusMap(currentValues, prevValues);
331 result = isHealthStatusChanged(result, currentValues, prevValues);
334 } else if (beHealthCheckInfos == null && prevBeHealthCheckInfos == null) {
337 writeLogAlarm(prevBeHealthCheckInfos);
343 private void writeLogAlarm(List<HealthCheckInfo> prevBeHealthCheckInfos) {
344 logAlarm(format(COMPONENT_CHANGED_MESSAGE, "", prevBeHealthCheckInfos == null ? "null" : "true",
345 prevBeHealthCheckInfos == null ? "true" : "null"));
348 private boolean isHealthStatusChanged(boolean result, Map<String, HealthCheckStatus> currentValues, Map<String, HealthCheckStatus> prevValues) {
349 for (Entry<String, HealthCheckStatus> entry : currentValues.entrySet()) {
350 String key = entry.getKey();
351 HealthCheckStatus value = entry.getValue();
352 if (!prevValues.containsKey(key)) {
353 result = true; //component missing
354 logAlarm(format(COMPONENT_CHANGED_MESSAGE, key, prevValues.get(key), currentValues.get(key)));
357 HealthCheckStatus prevHealthCheckStatus = prevValues.get(key);
358 if (value != prevHealthCheckStatus) {
359 result = true; //component status changed
360 logAlarm(format(COMPONENT_CHANGED_MESSAGE, key, prevValues.get(key), currentValues.get(key)));
367 private void updateHealthCheckStatusMap(Map<String, HealthCheckStatus> currentValues, Map<String, HealthCheckStatus> prevValues) {
368 Map<String, HealthCheckStatus> notPresent;
369 if (currentValues.keySet().containsAll(prevValues.keySet())) {
370 notPresent = new HashMap<>(currentValues);
371 notPresent.keySet().removeAll(prevValues.keySet());
373 notPresent = new HashMap<>(prevValues);
374 notPresent.keySet().removeAll(currentValues.keySet());
376 for (String component : notPresent.keySet()) {
377 logAlarm(format(COMPONENT_CHANGED_MESSAGE, component, prevValues.get(component), currentValues.get(component)));
381 private String buildOnBoardingHealthCheckUrl() {
382 Configuration.OnboardingConfig onboardingConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getOnboarding();
383 if (onboardingConfig != null) {
384 return String.format(hcUrl, onboardingConfig.getProtocol(), onboardingConfig.getHost(), onboardingConfig.getPort(),
385 onboardingConfig.getHealthCheckUri());
387 log.error("Onboarding health check configuration is missing.");
391 public class HealthCheckScheduledTask implements Runnable {
395 mdcFieldsHandler.addInfoForErrorAndDebugLogging(LOG_PARTNER_NAME);
396 Configuration config = ConfigurationManager.getConfigurationManager().getConfiguration();
397 log.trace("Executing BE Health Check Task");
398 List<HealthCheckInfo> currentBeHealthCheckInfos = getBeHealthCheckInfos();
399 boolean healthStatus = healthCheckUtil.getAggregateStatus(currentBeHealthCheckInfos, config.getHealthStatusExclude());
400 boolean prevHealthStatus = healthCheckUtil.getAggregateStatus(prevBeHealthCheckInfos, config.getHealthStatusExclude());
401 boolean anyStatusChanged = anyStatusChanged(currentBeHealthCheckInfos, prevBeHealthCheckInfos);
402 if (prevHealthStatus != healthStatus || anyStatusChanged) {
403 log.trace("BE Health State Changed to {}. Issuing alarm / recovery alarm...", healthStatus);
404 prevBeHealthCheckInfos = currentBeHealthCheckInfos;
405 logAlarm(healthStatus);
409 private void logAlarm(boolean prevHealthState) {
410 if (prevHealthState) {
411 BeEcompErrorManager.getInstance().logBeHealthCheckRecovery(BE_HEALTH_CHECK_STR);
413 BeEcompErrorManager.getInstance().logBeHealthCheckError(BE_HEALTH_CHECK_STR);