catalog-fe/src/main/java/org/openecomp/sdc/fe/servlets/HealthCheckService.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.fe.servlets;
  22
  23 import com.fasterxml.jackson.core.type.TypeReference;
  24 import com.fasterxml.jackson.databind.ObjectMapper;
  25 import com.google.gson.Gson;
  26 import com.google.gson.GsonBuilder;
  27 import com.google.gson.JsonSyntaxException;
  28 import com.google.gson.reflect.TypeToken;
  29 import org.openecomp.sdc.common.api.HealthCheckInfo;
  30 import org.openecomp.sdc.common.api.HealthCheckWrapper;
  31 import org.openecomp.sdc.common.config.EcompErrorEnum;
  32 import org.openecomp.sdc.common.http.client.api.HttpResponse;
  33 import org.openecomp.sdc.common.http.config.HttpClientConfig;
  34 import org.openecomp.sdc.common.http.config.Timeouts;
  35 import org.openecomp.sdc.common.util.HealthCheckUtil;
  36 import org.openecomp.sdc.fe.config.Configuration;
  37 import org.openecomp.sdc.fe.config.ConfigurationManager;
  38 import org.openecomp.sdc.fe.config.FeEcompErrorManager;
  39 import org.slf4j.Logger;
  40
  41 import javax.servlet.ServletContext;
  42 import javax.ws.rs.core.Response;
  43 import java.io.IOException;
  44 import java.lang.reflect.Type;
  45 import java.util.ArrayList;
  46 import java.util.List;
  47 import java.util.Map;
  48 import java.util.concurrent.ScheduledExecutorService;
  49 import java.util.concurrent.TimeUnit;
  50
  51 import static java.util.Arrays.asList;
  52 import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
  53 import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
  54 import static org.apache.http.HttpStatus.SC_OK;
  55 import static org.openecomp.sdc.common.api.Constants.CONFIGURATION_MANAGER_ATTR;
  56 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_BE;
  57 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_CASSANDRA;
  58 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_DCAE;
  59 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_DISTRIBUTION_ENGINE;
  60 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_FE;
  61 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_JANUSGRAPH;
  62 import static org.openecomp.sdc.common.api.Constants.HC_COMPONENT_ON_BOARDING;
  63 import static org.openecomp.sdc.common.api.Constants.HTTPS;
  64 import static org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus.DOWN;
  65 import static org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus.UNKNOWN;
  66 import static org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus.UP;
  67 import static org.openecomp.sdc.common.http.client.api.HttpRequest.get;
  68 import static org.openecomp.sdc.common.impl.ExternalConfiguration.getAppVersion;
  69 import static org.slf4j.LoggerFactory.getLogger;
  70
  71 public class HealthCheckService {
  72
  73     private static final String URL = "%s://%s:%s/sdc2/rest/healthCheck";
  74     private static final int HEALTH_STATUS_CODE = 500;
  75     private static Logger healthLogger = getLogger("asdc.fe.healthcheck");
  76     private static Logger log = getLogger(HealthCheckService.class.getName());
  77     private final List<String> healthCheckFeComponents = asList(HC_COMPONENT_ON_BOARDING, HC_COMPONENT_DCAE);
  78     private static final HealthCheckUtil healthCheckUtil = new HealthCheckUtil();
  79     private static final String DEBUG_CONTEXT = "HEALTH_FE";
  80     /**
  81      * This executor will execute the health check task.
  82      */
  83     private ScheduledExecutorService healthCheckExecutor = newSingleThreadScheduledExecutor((Runnable r) -> new Thread(r, "FE-Health-Check-Thread"));
  84
  85     public void setTask(HealthCheckScheduledTask task) {
  86         this.task = task;
  87     }
  88
  89     private HealthCheckScheduledTask task;
  90     private HealthStatus lastHealthStatus = new HealthStatus(HEALTH_STATUS_CODE, "{}");
  91     private ServletContext context;
  92
  93     public HealthCheckService(ServletContext context) {
  94         this.context = context;
  95         this.task = new HealthCheckScheduledTask();
  96     }
  97
  98     public void start(int interval) {
  99         this.healthCheckExecutor.scheduleAtFixedRate(getTask(), 0, interval, TimeUnit.SECONDS);
 100     }
 101
 102     /**
 103      * To be used by the HealthCheckServlet
 104      *
 105      * @return
 106      */
 107     public Response getFeHealth() {
 108         return this.buildResponse(lastHealthStatus.statusCode, lastHealthStatus.body);
 109     }
 110
 111     private Response buildResponse(int status, String jsonResponse) {
 112         healthLogger.trace("FE and BE health check status: {}", jsonResponse);
 113         return Response.status(status).entity(jsonResponse).build();
 114     }
 115
 116     public HealthStatus getLastHealthStatus() {
 117         return lastHealthStatus;
 118     }
 119
 120     public HealthCheckScheduledTask getTask() {
 121         return task;
 122     }
 123
 124     //immutable
 125     protected static class HealthStatus {
 126
 127         private String body;
 128         private int statusCode;
 129
 130         public HealthStatus(int code, String body) {
 131             this.body = body;
 132             this.statusCode = code;
 133         }
 134
 135         public int getStatusCode() {
 136             return statusCode;
 137         }
 138
 139         public String getBody() {
 140             return body;
 141         }
 142     }
 143
 144     protected class HealthCheckScheduledTask implements Runnable {
 145
 146         public static final int READ_TIMEOUT_DEFAULT_VAL = 5000;
 147         public static final int CONNECT_TIMEOUT_MS = 3000;
 148
 149         @Override
 150         public void run() {
 151             healthLogger.trace("Executing FE Health Check Task - Start");
 152             HealthStatus currentHealth = checkHealth();
 153             int currentHealthStatus = currentHealth.statusCode;
 154             healthLogger.trace("Executing FE Health Check Task - Status = {}", currentHealthStatus);
 155
 156             // In case health status was changed, issue alarm/recovery
 157             if (currentHealthStatus != lastHealthStatus.statusCode) {
 158                 log.trace("FE Health State Changed to {}. Issuing alarm / recovery alarm...", currentHealthStatus);
 159                 logFeAlarm(currentHealthStatus);
 160             }
 161
 162             // Anyway, update latest response
 163             lastHealthStatus = currentHealth;
 164         }
 165
 166         private List<HealthCheckInfo> addHostedComponentsFeHealthCheck(String baseComponent) {
 167             Configuration config = getConfig();
 168
 169             String healthCheckUrl = null;
 170             switch (baseComponent) {
 171                 case HC_COMPONENT_ON_BOARDING:
 172                     healthCheckUrl = buildOnboardingHealthCheckUrl(config);
 173                     break;
 174                 case HC_COMPONENT_DCAE:
 175                     healthCheckUrl = buildDcaeHealthCheckUrl(config);
 176                     break;
 177                 default:
 178                     log.debug("Unsupported base component {}", baseComponent);
 179             }
 180
 181             StringBuilder description = new StringBuilder("");
 182             int connectTimeoutMs = CONNECT_TIMEOUT_MS;
 183             int readTimeoutMs = config.getHealthCheckSocketTimeoutInMs(READ_TIMEOUT_DEFAULT_VAL);
 184
 185             if (healthCheckUrl != null) {
 186                 ObjectMapper mapper = new ObjectMapper();
 187                 try {
 188                     HttpResponse<String> response = get(healthCheckUrl, new HttpClientConfig(new Timeouts(connectTimeoutMs, readTimeoutMs)));
 189                     int beStatus = response.getStatusCode();
 190                     if (beStatus == SC_OK || beStatus == SC_INTERNAL_SERVER_ERROR) {
 191                         String beJsonResponse = response.getResponse();
 192                         return convertResponse(beJsonResponse, mapper, baseComponent, description, beStatus);
 193                     } else {
 194                         description.append("Response code: " + beStatus);
 195                         log.trace("{} Health Check Response code: {}", baseComponent, beStatus);
 196                     }
 197                 } catch (Exception e) {
 198                     log.error("{} Unexpected response ", baseComponent, e);
 199                     description.append(baseComponent + " Unexpected response: " + e.getMessage());
 200                 }
 201             } else {
 202                 description.append(baseComponent + " health check Configuration is missing");
 203             }
 204
 205             return asList(new HealthCheckInfo(HC_COMPONENT_FE, DOWN, null, description.toString()));
 206         }
 207
 208         private void logFeAlarm(int lastFeStatus) {
 209             switch (lastFeStatus) {
 210                 case SC_OK:
 211                     FeEcompErrorManager.getInstance().processEcompError(DEBUG_CONTEXT, EcompErrorEnum.FeHealthCheckRecovery, "FE Health Recovered");
 212                     FeEcompErrorManager.getInstance().logFeHealthCheckRecovery("FE Health Recovered");
 213                     break;
 214                 case SC_INTERNAL_SERVER_ERROR:
 215                     FeEcompErrorManager.getInstance().processEcompError(DEBUG_CONTEXT, EcompErrorEnum.FeHealthCheckError, "Connection with ASDC-BE is probably down");
 216                     FeEcompErrorManager.getInstance().logFeHealthCheckError("Connection with ASDC-BE is probably down");
 217                     break;
 218                 default:
 219                     break;
 220             }
 221         }
 222
 223         protected HealthStatus checkHealth() {
 224             HttpResponse<String> response;
 225             try {
 226                 Gson gson = new GsonBuilder().setPrettyPrinting().create();
 227                 Configuration config = getConfig();
 228                 String redirectedUrl = String.format(URL, config.getBeProtocol(), config.getBeHost(),
 229                         HTTPS.equals(config.getBeProtocol()) ? config.getBeSslPort() : config.getBeHttpPort());
 230
 231                 int connectTimeoutMs = CONNECT_TIMEOUT_MS;
 232                 int readTimeoutMs = config.getHealthCheckSocketTimeoutInMs(READ_TIMEOUT_DEFAULT_VAL);
 233
 234                 HealthCheckWrapper feAggHealthCheck;
 235                 try {
 236                     response = get(redirectedUrl, new HttpClientConfig(new Timeouts(connectTimeoutMs, readTimeoutMs)));
 237                     log.debug("HC call to BE - status code is {}", response.getStatusCode());
 238                     String beJsonResponse = response.getResponse();
 239                     feAggHealthCheck = getFeHealthCheckInfos(gson, beJsonResponse);
 240                 } catch (Exception e) {
 241                     log.debug("Health Check error when trying to connect to BE or external FE. Error: {}", e.getMessage());
 242                     log.error("Health Check error when trying to connect to BE or external FE.", e);
 243                     String beDowneResponse = gson.toJson(getBeDownCheckInfos());
 244                     return new HealthStatus(SC_INTERNAL_SERVER_ERROR, beDowneResponse);
 245                 }
 246
 247                 //Getting aggregate FE status
 248                 boolean aggregateFeStatus = (response != null && response.getStatusCode() == SC_INTERNAL_SERVER_ERROR) ? false : healthCheckUtil.getAggregateStatus(feAggHealthCheck.getComponentsInfo(), config.getHealthStatusExclude());
 249                 return new HealthStatus(aggregateFeStatus ? SC_OK : SC_INTERNAL_SERVER_ERROR, gson.toJson(feAggHealthCheck));
 250             } catch (Exception e) {
 251                 FeEcompErrorManager.getInstance().processEcompError(DEBUG_CONTEXT, EcompErrorEnum.FeHealthCheckGeneralError, "Unexpected FE Health check error");
 252                 FeEcompErrorManager.getInstance().logFeHealthCheckGeneralError("Unexpected FE Health check error");
 253                 log.error("Unexpected FE health check error {}", e.getMessage());
 254                 return new HealthStatus(SC_INTERNAL_SERVER_ERROR, e.getMessage());
 255             }
 256         }
 257
 258         protected Configuration getConfig() {
 259             return ((ConfigurationManager) context.getAttribute(CONFIGURATION_MANAGER_ATTR))
 260                     .getConfiguration();
 261         }
 262
 263         protected HealthCheckWrapper getFeHealthCheckInfos(Gson gson, String responseString) {
 264             Configuration config = getConfig();
 265             Type wrapperType = new TypeToken<HealthCheckWrapper>() {
 266             }.getType();
 267             HealthCheckWrapper healthCheckWrapper = gson.fromJson(responseString, wrapperType);
 268             String appVersion = getAppVersion();
 269             String description = "OK";
 270             healthCheckWrapper.getComponentsInfo()
 271                     .add(new HealthCheckInfo(HC_COMPONENT_FE, UP, appVersion, description));
 272
 273             //add hosted components fe component
 274             for (String component : healthCheckFeComponents) {
 275                 List<HealthCheckInfo> feComponentsInfo = addHostedComponentsFeHealthCheck(component);
 276                 HealthCheckInfo baseComponentHCInfo = healthCheckWrapper.getComponentsInfo().stream().filter(c -> c.getHealthCheckComponent().equals(component)).findFirst().orElse(null);
 277                 if (baseComponentHCInfo != null) {
 278                     if (baseComponentHCInfo.getComponentsInfo() == null) {
 279                         baseComponentHCInfo.setComponentsInfo(new ArrayList<>());
 280                     }
 281                     baseComponentHCInfo.getComponentsInfo().addAll(feComponentsInfo);
 282                     boolean status = healthCheckUtil.getAggregateStatus(baseComponentHCInfo.getComponentsInfo(), config.getHealthStatusExclude());
 283                     baseComponentHCInfo.setHealthCheckStatus(status ? UP : DOWN);
 284
 285                     String componentsDesc = healthCheckUtil.getAggregateDescription(baseComponentHCInfo.getComponentsInfo(), baseComponentHCInfo.getDescription());
 286                     if (componentsDesc.length() > 0) { //aggregated description contains all the internal components desc
 287                         baseComponentHCInfo.setDescription(componentsDesc);
 288                     }
 289                 } else {
 290                     log.error("{} not exists in HealthCheck info", component);
 291                 }
 292             }
 293             return healthCheckWrapper;
 294         }
 295
 296         private HealthCheckWrapper getBeDownCheckInfos() {
 297             List<HealthCheckInfo> healthCheckInfos = new ArrayList<>();
 298             healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_FE, UP,
 299                     getAppVersion(), "OK"));
 300             healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, null));
 301             healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_JANUSGRAPH, UNKNOWN, null, null));
 302             healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_CASSANDRA, UNKNOWN, null, null));
 303             healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_DISTRIBUTION_ENGINE, UNKNOWN, null, null));
 304             healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_ON_BOARDING, UNKNOWN, null, null));
 305             healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_DCAE, UNKNOWN, null, null));
 306             return new HealthCheckWrapper(healthCheckInfos, "UNKNOWN", "UNKNOWN");
 307         }
 308
 309         private String buildOnboardingHealthCheckUrl(Configuration config) {
 310
 311             Configuration.OnboardingConfig onboardingConfig = config.getOnboarding();
 312
 313             if (onboardingConfig != null) {
 314                 String protocol = onboardingConfig.getProtocolFe();
 315                 String host = onboardingConfig.getHostFe();
 316                 Integer port = onboardingConfig.getPortFe();
 317                 String uri = onboardingConfig.getHealthCheckUriFe();
 318
 319                 return protocol + "://" + host + ":" + port + uri;
 320             }
 321
 322             log.error("onboarding health check configuration is missing.");
 323             return null;
 324         }
 325
 326         private String buildDcaeHealthCheckUrl(Configuration config) {
 327
 328             Configuration.DcaeConfig dcaeConfig = config.getDcae();
 329
 330             if (dcaeConfig != null) {
 331                 String protocol = dcaeConfig.getProtocol();
 332                 String host = dcaeConfig.getHost();
 333                 Integer port = dcaeConfig.getPort();
 334                 String uri = dcaeConfig.getHealthCheckUri();
 335
 336                 return protocol + "://" + host + ":" + port + uri;
 337             }
 338
 339             log.error("dcae health check configuration is missing.");
 340             return null;
 341         }
 342
 343         private List<HealthCheckInfo> convertResponse(String beJsonResponse, ObjectMapper mapper, String baseComponent, StringBuilder description, int beStatus) {
 344             try {
 345                 Map<String, Object> healthCheckMap = mapper.readValue(beJsonResponse, new TypeReference<Map<String, Object>>() {
 346                 });
 347                 if (healthCheckMap.containsKey("componentsInfo")) {
 348                     return mapper.convertValue(healthCheckMap.get("componentsInfo"), new TypeReference<List<HealthCheckInfo>>() {
 349                     });
 350                 } else {
 351                     description.append("Internal components are missing");
 352                 }
 353             } catch (JsonSyntaxException | IOException e) {
 354                 log.error("{} Unexpected response body ", baseComponent, e);
 355                 description.append(baseComponent + " Unexpected response body. Response code: " + beStatus);
 356             }
 357             return new ArrayList<>();
 358         }
 359     }
 360
 361 }