catalog-be/src/main/java/org/openecomp/sdc/be/components/distribution/engine/DistributionEngineClusterHealth.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.be.components.distribution.engine;
  22
  23 import org.openecomp.sdc.be.config.BeEcompErrorManager;
  24 import org.openecomp.sdc.be.config.ConfigurationManager;
  25 import org.openecomp.sdc.be.config.DistributionEngineConfiguration;
  26 import org.openecomp.sdc.common.api.Constants;
  27 import org.openecomp.sdc.common.api.HealthCheckInfo;
  28 import org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus;
  29 import org.openecomp.sdc.common.config.EcompErrorName;
  30 import org.slf4j.Logger;
  31 import org.slf4j.LoggerFactory;
  32 import org.springframework.stereotype.Component;
  33
  34 import javax.annotation.PostConstruct;
  35 import javax.annotation.PreDestroy;
  36 import java.util.ArrayList;
  37 import java.util.Collection;
  38 import java.util.List;
  39 import java.util.Map;
  40 import java.util.concurrent.*;
  41 import java.util.concurrent.atomic.AtomicBoolean;
  42
  43 @Component("distribution-engine-cluster-health")
  44 public class DistributionEngineClusterHealth {
  45
  46     protected static String UEB_HEALTH_LOG_CONTEXT = "ueb.healthcheck";
  47
  48     private static final Logger healthLogger = LoggerFactory.getLogger(UEB_HEALTH_LOG_CONTEXT);
  49
  50     private static final String UEB_HEALTH_CHECK_STR = "uebHealthCheck";
  51
  52     boolean lastHealthState = false;
  53
  54     Object lockOject = new Object();
  55
  56     private long reconnectInterval = 5;
  57
  58     private long healthCheckReadTimeout = 20;
  59
  60     private static final Logger logger = LoggerFactory.getLogger(DistributionEngineClusterHealth.class);
  61
  62     private List<String> uebServers = null;
  63
  64     private String publicApiKey = null;
  65
  66     public enum HealthCheckInfoResult {
  67
  68         OK(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.UP, null, ClusterStatusDescription.OK.getDescription())),
  69         UNAVAILABLE(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.DOWN, null, ClusterStatusDescription.UNAVAILABLE.getDescription())),
  70         NOT_CONFIGURED(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.DOWN, null, ClusterStatusDescription.NOT_CONFIGURED.getDescription())),
  71         DISABLED(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.DOWN, null, ClusterStatusDescription.DISABLED.getDescription()));
  72
  73         private HealthCheckInfo healthCheckInfo;
  74
  75         HealthCheckInfoResult(HealthCheckInfo healthCheckInfo) {
  76             this.healthCheckInfo = healthCheckInfo;
  77         }
  78
  79         public HealthCheckInfo getHealthCheckInfo() {
  80             return healthCheckInfo;
  81         }
  82
  83     }
  84
  85     private HealthCheckInfo healthCheckInfo = HealthCheckInfoResult.UNAVAILABLE.getHealthCheckInfo();
  86
  87     private Map<String, AtomicBoolean> envNamePerStatus = null;
  88
  89     private ScheduledFuture<?> scheduledFuture = null;
  90
  91     ScheduledExecutorService healthCheckScheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
  92         @Override
  93         public Thread newThread(Runnable r) {
  94             return new Thread(r, "UEB-Health-Check-Task");
  95         }
  96     });
  97
  98     HealthCheckScheduledTask healthCheckScheduledTask = null;
  99
 100     public enum ClusterStatusDescription {
 101
 102         OK("OK"), UNAVAILABLE("U-EB cluster is not available"), NOT_CONFIGURED("U-EB cluster is not configured"), DISABLED("DE is disabled in configuration");
 103
 104         private String desc;
 105
 106         ClusterStatusDescription(String desc) {
 107             this.desc = desc;
 108         }
 109
 110         public String getDescription() {
 111             return desc;
 112         }
 113
 114     }
 115
 116     /**
 117      * Health Check Task Scheduler.
 118      *
 119      * It schedules a task which send a apiKey get query towards the UEB servers. In case a query to the first UEB server is failed, then a second query is sent to the next UEB server.
 120      *
 121      *
 122      * @author esofer
 123      *
 124      */
 125     public class HealthCheckScheduledTask implements Runnable {
 126
 127         List<UebHealthCheckCall> healthCheckCalls = new ArrayList<>();
 128
 129         public HealthCheckScheduledTask(List<String> uebServers) {
 130
 131             logger.debug("Create health check calls for servers {}", uebServers);
 132             if (uebServers != null) {
 133                 for (String server : uebServers) {
 134                     healthCheckCalls.add(new UebHealthCheckCall(server, publicApiKey));
 135                 }
 136             }
 137         }
 138
 139         @Override
 140         public void run() {
 141
 142             healthLogger.trace("Executing UEB Health Check Task - Start");
 143
 144             boolean healthStatus = verifyAtLeastOneEnvIsUp();
 145
 146             if (true == healthStatus) {
 147                 boolean queryUebStatus = queryUeb();
 148                 if (queryUebStatus == lastHealthState) {
 149                     return;
 150                 }
 151
 152                 synchronized (lockOject) {
 153                     if (queryUebStatus != lastHealthState) {
 154                         logger.trace("UEB Health State Changed to {}. Issuing alarm / recovery alarm...", healthStatus);
 155                         lastHealthState = queryUebStatus;
 156                         logAlarm(lastHealthState);
 157                         if (true == queryUebStatus) {
 158                             healthCheckInfo = HealthCheckInfoResult.OK.getHealthCheckInfo();
 159                         } else {
 160                             healthCheckInfo = HealthCheckInfoResult.UNAVAILABLE.getHealthCheckInfo();
 161                         }
 162                     }
 163                 }
 164             } else {
 165                 healthLogger.trace("Not all UEB Environments are up");
 166             }
 167
 168         }
 169
 170         /**
 171          * verify that at least one environment is up.
 172          *
 173          */
 174         private boolean verifyAtLeastOneEnvIsUp() {
 175
 176             boolean healthStatus = false;
 177
 178             if (envNamePerStatus != null) {
 179                 Collection<AtomicBoolean> values = envNamePerStatus.values();
 180                 if (values != null) {
 181                     for (AtomicBoolean status : values) {
 182                         if (true == status.get()) {
 183                             healthStatus = true;
 184                             break;
 185                         }
 186                     }
 187                 }
 188             }
 189
 190             return healthStatus;
 191         }
 192
 193         /**
 194          * executor for the query itself
 195          */
 196         ExecutorService healthCheckExecutor = Executors.newSingleThreadExecutor(new ThreadFactory() {
 197             @Override
 198             public Thread newThread(Runnable r) {
 199                 return new Thread(r, "UEB-Health-Check-Thread");
 200             }
 201         });
 202
 203         /**
 204          * go all UEB servers and send a get apiKeys query. In case a query is succeed, no query is sent to the rest of UEB servers.
 205          *
 206          *
 207          * @return
 208          */
 209         private boolean queryUeb() {
 210
 211             Boolean result = false;
 212             int retryNumber = 1;
 213             for (UebHealthCheckCall healthCheckCall : healthCheckCalls) {
 214                 try {
 215
 216                     healthLogger.debug("Before running Health Check retry query number {} towards UEB server {}", retryNumber, healthCheckCall.getServer());
 217
 218                     Future<Boolean> future = healthCheckExecutor.submit(healthCheckCall);
 219                     result = future.get(healthCheckReadTimeout, TimeUnit.SECONDS);
 220
 221                     healthLogger.debug("After running Health Check retry query number {} towards UEB server {}. Result is {}", retryNumber, healthCheckCall.getServer(), result);
 222
 223                     if (result != null && true == result.booleanValue()) {
 224                         break;
 225                     }
 226
 227                 } catch (Exception e) {
 228                     String message = e.getMessage();
 229                     if (message == null) {
 230                         message = e.getClass().getName();
 231                     }
 232                     healthLogger.debug("Error occured during running Health Check retry query towards UEB server {}. Result is {}", healthCheckCall.getServer(), message);
 233                     healthLogger.trace("Error occured during running Health Check retry query towards UEB server {}. Result is {}", healthCheckCall.getServer(), message, e);
 234                 }
 235                 retryNumber++;
 236
 237             }
 238
 239             return result;
 240
 241         }
 242
 243         public List<UebHealthCheckCall> getHealthCheckCalls() {
 244             return healthCheckCalls;
 245         }
 246
 247     }
 248
 249     @PostConstruct
 250     protected void init() {
 251
 252         logger.trace("Enter init method of DistributionEngineClusterHealth");
 253
 254         Long reconnectIntervalConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getUebHealthCheckReconnectIntervalInSeconds();
 255         if (reconnectIntervalConfig != null) {
 256             reconnectInterval = reconnectIntervalConfig.longValue();
 257         }
 258         Long healthCheckReadTimeoutConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getUebHealthCheckReadTimeout();
 259         if (healthCheckReadTimeoutConfig != null) {
 260             healthCheckReadTimeout = healthCheckReadTimeoutConfig.longValue();
 261         }
 262
 263         DistributionEngineConfiguration distributionEngineConfiguration = ConfigurationManager.getConfigurationManager().getDistributionEngineConfiguration();
 264
 265         this.uebServers = distributionEngineConfiguration.getUebServers();
 266         this.publicApiKey = distributionEngineConfiguration.getUebPublicKey();
 267
 268         this.healthCheckScheduledTask = new HealthCheckScheduledTask(this.uebServers);
 269
 270         logger.trace("Exit init method of DistributionEngineClusterHealth");
 271
 272     }
 273
 274     @PreDestroy
 275     protected void destroy() {
 276
 277         if (scheduledFuture != null) {
 278             scheduledFuture.cancel(true);
 279             scheduledFuture = null;
 280         }
 281
 282         if (healthCheckScheduler != null) {
 283             healthCheckScheduler.shutdown();
 284         }
 285
 286     }
 287
 288     /**
 289      * Start health check task.
 290      *
 291      * @param envNamePerStatus
 292      * @param startTask
 293      */
 294     public void startHealthCheckTask(Map<String, AtomicBoolean> envNamePerStatus, boolean startTask) {
 295         this.envNamePerStatus = envNamePerStatus;
 296
 297         if (startTask == true && this.scheduledFuture == null) {
 298             this.scheduledFuture = this.healthCheckScheduler.scheduleAtFixedRate(healthCheckScheduledTask, 0, reconnectInterval, TimeUnit.SECONDS);
 299         }
 300     }
 301
 302     public void startHealthCheckTask(Map<String, AtomicBoolean> envNamePerStatus) {
 303         startHealthCheckTask(envNamePerStatus, true);
 304     }
 305
 306     private void logAlarm(boolean lastHealthState) {
 307         if (lastHealthState == true) {
 308             BeEcompErrorManager.getInstance().logBeHealthCheckUebClusterRecovery(UEB_HEALTH_CHECK_STR);
 309         } else {
 310             BeEcompErrorManager.getInstance().logBeHealthCheckUebClusterError(UEB_HEALTH_CHECK_STR);
 311         }
 312     }
 313
 314     public HealthCheckInfo getHealthCheckInfo() {
 315         return healthCheckInfo;
 316     }
 317
 318     /**
 319      * change the health check to DISABLE
 320      */
 321     public void setHealthCheckUebIsDisabled() {
 322         healthCheckInfo = HealthCheckInfoResult.DISABLED.getHealthCheckInfo();
 323     }
 324
 325     /**
 326      * change the health check to NOT CONFGIURED
 327      */
 328     public void setHealthCheckUebConfigurationError() {
 329         healthCheckInfo = HealthCheckInfoResult.NOT_CONFIGURED.getHealthCheckInfo();
 330     }
 331
 332     public void setHealthCheckOkAndReportInCaseLastStateIsDown() {
 333
 334         if (lastHealthState == true) {
 335             return;
 336         }
 337         synchronized (lockOject) {
 338             if (lastHealthState == false) {
 339                 logger.debug("Going to update health check state to available");
 340                 lastHealthState = true;
 341                 healthCheckInfo = HealthCheckInfoResult.OK.getHealthCheckInfo();
 342                 logAlarm(lastHealthState);
 343             }
 344         }
 345
 346     }
 347
 348 }