catalog-fe/src/main/java/org/openecomp/sdc/fe/servlets/HealthCheckService.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.fe.servlets;
  22
  23 import com.fasterxml.jackson.core.type.TypeReference;
  24 import com.fasterxml.jackson.databind.ObjectMapper;
  25 import com.google.gson.Gson;
  26 import com.google.gson.GsonBuilder;
  27 import com.google.gson.JsonSyntaxException;
  28 import com.google.gson.reflect.TypeToken;
  29 import org.openecomp.sdc.common.api.HealthCheckInfo;
  30 import org.openecomp.sdc.common.api.HealthCheckWrapper;
  31 import org.openecomp.sdc.common.config.EcompErrorEnum;
  32 import org.openecomp.sdc.common.http.client.api.HttpResponse;
  33 import org.openecomp.sdc.common.http.config.HttpClientConfig;
  34 import org.openecomp.sdc.common.http.config.Timeouts;
  35 import org.openecomp.sdc.common.util.HealthCheckUtil;
  36 import org.openecomp.sdc.fe.config.Configuration;
  37 import org.openecomp.sdc.fe.config.ConfigurationManager;
  38 import org.openecomp.sdc.fe.config.FeEcompErrorManager;
  39 import org.slf4j.Logger;
  40
  41 import javax.servlet.ServletContext;
  42 import javax.ws.rs.core.Response;
  43 import java.io.IOException;
  44 import java.lang.reflect.Type;
  45 import java.util.ArrayList;
  46 import java.util.List;
  47 import java.util.Map;
  48 import java.util.concurrent.ScheduledExecutorService;
  49 import java.util.concurrent.TimeUnit;
  50
  51 import static java.util.Arrays.asList;
  52 import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
  53 import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
  54 import static org.apache.http.HttpStatus.SC_OK;
  55 import static org.openecomp.sdc.common.api.Constants.*;
  56 import static org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus.*;
  57 import static org.openecomp.sdc.common.http.client.api.HttpRequest.get;
  58 import static org.openecomp.sdc.common.impl.ExternalConfiguration.getAppVersion;
  59 import static org.slf4j.LoggerFactory.getLogger;
  60
  61 public class HealthCheckService {
  62
  63     private static final String URL = "%s://%s:%s/sdc2/rest/healthCheck";
  64     private static Logger healthLogger = getLogger("asdc.fe.healthcheck");
  65     private static Logger log = getLogger(HealthCheckService.class.getName());
  66     private final List<String> healthCheckFeComponents = asList(HC_COMPONENT_ON_BOARDING, HC_COMPONENT_DCAE);
  67     private static final HealthCheckUtil healthCheckUtil = new HealthCheckUtil();
  68     private static final String DEBUG_CONTEXT = "HEALTH_FE";
  69     /**
  70      * This executor will execute the health check task.
  71      */
  72     ScheduledExecutorService healthCheckExecutor = newSingleThreadScheduledExecutor((Runnable r) -> new Thread(r, "FE-Health-Check-Thread"));
  73
  74     public void setTask(HealthCheckScheduledTask task) {
  75         this.task = task;
  76     }
  77
  78     private HealthCheckScheduledTask task ;
  79     private HealthStatus lastHealthStatus = new HealthStatus(500, "{}");
  80     private ServletContext context;
  81
  82     public HealthCheckService(ServletContext context) {
  83         this.context = context;
  84         this.task = new HealthCheckScheduledTask();
  85     }
  86
  87     public void start(int interval) {
  88         this.healthCheckExecutor.scheduleAtFixedRate( getTask() , 0, interval, TimeUnit.SECONDS);
  89     }
  90
  91     /**
  92      * To be used by the HealthCheckServlet
  93      *
  94      * @return
  95      */
  96     public Response getFeHealth() {
  97         return this.buildResponse(lastHealthStatus.statusCode, lastHealthStatus.body);
  98     }
  99
 100     private Response buildResponse(int status, String jsonResponse) {
 101         healthLogger.trace("FE and BE health check status: {}", jsonResponse);
 102         return Response.status(status).entity(jsonResponse).build();
 103     }
 104
 105     public HealthStatus getLastHealthStatus() {
 106         return lastHealthStatus;
 107     }
 108     public HealthCheckScheduledTask getTask() {
 109         return task;
 110     }
 111
 112     //immutable
 113     protected static class HealthStatus {
 114
 115         private String body;
 116         private int statusCode;
 117
 118         public HealthStatus(int code, String body) {
 119             this.body = body;
 120             this.statusCode = code;
 121         }
 122
 123         public int getStatusCode() {
 124             return statusCode;
 125         }
 126
 127         public String getBody() {
 128             return body;
 129         }
 130     }
 131
 132     protected class HealthCheckScheduledTask implements Runnable {
 133         @Override
 134         public void run() {
 135             healthLogger.trace("Executing FE Health Check Task - Start");
 136             HealthStatus currentHealth = checkHealth();
 137             int currentHealthStatus = currentHealth.statusCode;
 138             healthLogger.trace("Executing FE Health Check Task - Status = {}", currentHealthStatus);
 139
 140             // In case health status was changed, issue alarm/recovery
 141             if (currentHealthStatus != lastHealthStatus.statusCode) {
 142                 log.trace("FE Health State Changed to {}. Issuing alarm / recovery alarm...", currentHealthStatus);
 143                 logFeAlarm(currentHealthStatus);
 144             }
 145
 146             // Anyway, update latest response
 147             lastHealthStatus = currentHealth;
 148         }
 149
 150         private List<HealthCheckInfo> addHostedComponentsFeHealthCheck(String baseComponent) {
 151             Configuration config = getConfig();
 152
 153             String healthCheckUrl = null;
 154             switch (baseComponent) {
 155                 case HC_COMPONENT_ON_BOARDING:
 156                     healthCheckUrl = buildOnboardingHealthCheckUrl(config);
 157                     break;
 158                 case HC_COMPONENT_DCAE:
 159                     healthCheckUrl = buildDcaeHealthCheckUrl(config);
 160                     break;
 161                 default:
 162                     log.debug("Unsupported base component {}", baseComponent);
 163             }
 164
 165             StringBuilder description = new StringBuilder("");
 166             int connectTimeoutMs = 3000;
 167             int readTimeoutMs = config.getHealthCheckSocketTimeoutInMs(5000);
 168
 169             if (healthCheckUrl != null) {
 170                 ObjectMapper mapper = new ObjectMapper();
 171                 try {
 172                     HttpResponse<String> response = get(healthCheckUrl, new HttpClientConfig(new Timeouts(connectTimeoutMs, readTimeoutMs)));
 173                     int beStatus = response.getStatusCode();
 174                     if (beStatus == SC_OK || beStatus == SC_INTERNAL_SERVER_ERROR) {
 175                         String beJsonResponse = response.getResponse();
 176                         return convertResponse(beJsonResponse, mapper, baseComponent, description, beStatus);
 177                     } else {
 178                         description.append("Response code: " + beStatus);
 179                         log.trace("{} Health Check Response code: {}", baseComponent, beStatus);
 180                     }
 181                 } catch (Exception e) {
 182                     log.error("{} Unexpected response ", baseComponent, e);
 183                     description.append(baseComponent + " Unexpected response: " + e.getMessage());
 184                 }
 185             } else {
 186                 description.append(baseComponent + " health check Configuration is missing");
 187             }
 188
 189             return asList(new HealthCheckInfo(HC_COMPONENT_FE, DOWN, null, description.toString()));
 190         }
 191
 192         private void logFeAlarm(int lastFeStatus) {
 193             switch (lastFeStatus) {
 194                 case 200:
 195                     FeEcompErrorManager.getInstance().processEcompError(DEBUG_CONTEXT, EcompErrorEnum.FeHealthCheckRecovery, "FE Health Recovered");
 196                     FeEcompErrorManager.getInstance().logFeHealthCheckRecovery("FE Health Recovered");
 197                     break;
 198                 case 500:
 199                     FeEcompErrorManager.getInstance().processEcompError(DEBUG_CONTEXT, EcompErrorEnum.FeHealthCheckError, "Connection with ASDC-BE is probably down");
 200                     FeEcompErrorManager.getInstance().logFeHealthCheckError("Connection with ASDC-BE is probably down");
 201                     break;
 202                 default:
 203                     break;
 204             }
 205         }
 206
 207         protected HealthStatus checkHealth() {
 208             HttpResponse<String> response;
 209             try {
 210                 Gson gson = new GsonBuilder().setPrettyPrinting().create();
 211                 Configuration config = getConfig();
 212                 String redirectedUrl = String.format(URL, config.getBeProtocol(), config.getBeHost(),
 213                         HTTPS.equals(config.getBeProtocol()) ? config.getBeSslPort() : config.getBeHttpPort());
 214
 215                 int connectTimeoutMs = 3000;
 216                 int readTimeoutMs = config.getHealthCheckSocketTimeoutInMs(5000);
 217
 218                 HealthCheckWrapper feAggHealthCheck;
 219                 try {
 220                     response = get(redirectedUrl, new HttpClientConfig(new Timeouts(connectTimeoutMs, readTimeoutMs)));
 221                     log.debug("HC call to BE - status code is {}", response.getStatusCode());
 222                     String beJsonResponse = response.getResponse();
 223                     feAggHealthCheck = getFeHealthCheckInfos(gson, beJsonResponse);
 224                 } catch (Exception e) {
 225                     log.debug("Health Check error when trying to connect to BE or external FE. Error: {}", e.getMessage());
 226                     log.error("Health Check error when trying to connect to BE or external FE.", e);
 227                     String beDowneResponse = gson.toJson(getBeDownCheckInfos());
 228                     return new HealthStatus(SC_INTERNAL_SERVER_ERROR, beDowneResponse);
 229                 }
 230
 231                 //Getting aggregate FE status
 232                 boolean aggregateFeStatus = (response != null && response.getStatusCode() == SC_INTERNAL_SERVER_ERROR) ? false : healthCheckUtil.getAggregateStatus(feAggHealthCheck.getComponentsInfo(), config.getHealthStatusExclude());
 233                 return new HealthStatus(aggregateFeStatus ? SC_OK : SC_INTERNAL_SERVER_ERROR, gson.toJson(feAggHealthCheck));
 234             } catch (Exception e) {
 235                 FeEcompErrorManager.getInstance().processEcompError(DEBUG_CONTEXT,EcompErrorEnum.FeHealthCheckGeneralError, "Unexpected FE Health check error");
 236                 FeEcompErrorManager.getInstance().logFeHealthCheckGeneralError("Unexpected FE Health check error");
 237                 log.error("Unexpected FE health check error {}", e.getMessage());
 238                 return new HealthStatus(SC_INTERNAL_SERVER_ERROR, e.getMessage());
 239             }
 240         }
 241
 242         protected Configuration getConfig(){
 243             return ((ConfigurationManager) context.getAttribute(CONFIGURATION_MANAGER_ATTR))
 244                     .getConfiguration();
 245         }
 246
 247         protected HealthCheckWrapper getFeHealthCheckInfos(Gson gson, String responseString) {
 248             Configuration config = getConfig();
 249             Type wrapperType = new TypeToken<HealthCheckWrapper>() {
 250             }.getType();
 251             HealthCheckWrapper healthCheckWrapper = gson.fromJson(responseString, wrapperType);
 252             String appVersion = getAppVersion();
 253             String description = "OK";
 254             healthCheckWrapper.getComponentsInfo()
 255                     .add(new HealthCheckInfo(HC_COMPONENT_FE, UP, appVersion, description));
 256
 257             //add hosted components fe component
 258             for (String component : healthCheckFeComponents) {
 259                 List<HealthCheckInfo> feComponentsInfo = addHostedComponentsFeHealthCheck(component);
 260                 HealthCheckInfo baseComponentHCInfo = healthCheckWrapper.getComponentsInfo().stream().filter(c -> c.getHealthCheckComponent().equals(component)).findFirst().orElse(null);
 261                 if (baseComponentHCInfo != null) {
 262                     if (baseComponentHCInfo.getComponentsInfo() == null) {
 263                         baseComponentHCInfo.setComponentsInfo(new ArrayList<>());
 264                     }
 265                     baseComponentHCInfo.getComponentsInfo().addAll(feComponentsInfo);
 266                     boolean status = healthCheckUtil.getAggregateStatus(baseComponentHCInfo.getComponentsInfo() ,config.getHealthStatusExclude());
 267                     baseComponentHCInfo.setHealthCheckStatus(status ? UP : DOWN);
 268
 269                     String componentsDesc = healthCheckUtil.getAggregateDescription(baseComponentHCInfo.getComponentsInfo(), baseComponentHCInfo.getDescription());
 270                     if (componentsDesc.length() > 0) { //aggregated description contains all the internal components desc
 271                         baseComponentHCInfo.setDescription(componentsDesc);
 272                     }
 273                 } else {
 274                     log.error("{} not exists in HealthCheck info", component);
 275                 }
 276             }
 277             return healthCheckWrapper;
 278         }
 279
 280         private HealthCheckWrapper getBeDownCheckInfos() {
 281             List<HealthCheckInfo> healthCheckInfos = new ArrayList<>();
 282             healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_FE, UP,
 283                     getAppVersion(), "OK"));
 284             healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_BE, DOWN, null, null));
 285             healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_TITAN, UNKNOWN, null, null));
 286             healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_CASSANDRA, UNKNOWN, null, null));
 287             healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_DISTRIBUTION_ENGINE, UNKNOWN, null, null));
 288             healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_ON_BOARDING, UNKNOWN, null, null));
 289             healthCheckInfos.add(new HealthCheckInfo(HC_COMPONENT_DCAE, UNKNOWN, null, null));
 290             return new HealthCheckWrapper(healthCheckInfos, "UNKNOWN", "UNKNOWN");
 291         }
 292
 293         private String buildOnboardingHealthCheckUrl(Configuration config) {
 294
 295             Configuration.OnboardingConfig onboardingConfig = config.getOnboarding();
 296
 297             if (onboardingConfig != null) {
 298                 String protocol = onboardingConfig.getProtocolFe();
 299                 String host = onboardingConfig.getHostFe();
 300                 Integer port = onboardingConfig.getPortFe();
 301                 String uri = onboardingConfig.getHealthCheckUriFe();
 302
 303                 return protocol + "://" + host + ":" + port + uri;
 304             }
 305
 306             log.error("onboarding health check configuration is missing.");
 307             return null;
 308         }
 309
 310         private String buildDcaeHealthCheckUrl(Configuration config) {
 311
 312             Configuration.DcaeConfig dcaeConfig = config.getDcae();
 313
 314             if (dcaeConfig != null) {
 315                 String protocol = dcaeConfig.getProtocol();
 316                 String host = dcaeConfig.getHost();
 317                 Integer port = dcaeConfig.getPort();
 318                 String uri = dcaeConfig.getHealthCheckUri();
 319
 320                 return protocol + "://" + host + ":" + port + uri;
 321             }
 322
 323             log.error("dcae health check configuration is missing.");
 324             return null;
 325         }
 326
 327         private List<HealthCheckInfo> convertResponse(String beJsonResponse, ObjectMapper mapper, String baseComponent, StringBuilder description, int beStatus) {
 328             try {
 329                 Map<String, Object> healthCheckMap = mapper.readValue(beJsonResponse, new TypeReference<Map<String, Object>>() {
 330                 });
 331                 if (healthCheckMap.containsKey("componentsInfo")) {
 332                     return mapper.convertValue(healthCheckMap.get("componentsInfo"), new TypeReference<List<HealthCheckInfo>>() {
 333                     });
 334                 } else {
 335                     description.append("Internal components are missing");
 336                 }
 337             } catch (JsonSyntaxException | IOException e) {
 338                 log.error("{} Unexpected response body ", baseComponent, e);
 339                 description.append(baseComponent + " Unexpected response body. Response code: " + beStatus);
 340             }
 341             return new ArrayList<>();
 342         }
 343     }
 344
 345 }