catalog-be/src/main/java/org/openecomp/sdc/be/components/distribution/engine/DistributionEngineClusterHealth.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.be.components.distribution.engine;
  22
  23 import org.openecomp.sdc.be.config.BeEcompErrorManager;
  24 import org.openecomp.sdc.be.config.ConfigurationManager;
  25 import org.openecomp.sdc.be.config.DistributionEngineConfiguration;
  26 import org.openecomp.sdc.common.api.Constants;
  27 import org.openecomp.sdc.common.api.HealthCheckInfo;
  28 import org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus;
  29 import org.openecomp.sdc.common.log.wrappers.Logger;
  30 import org.springframework.stereotype.Component;
  31
  32 import javax.annotation.PostConstruct;
  33 import javax.annotation.PreDestroy;
  34 import java.util.ArrayList;
  35 import java.util.Collection;
  36 import java.util.List;
  37 import java.util.Map;
  38 import java.util.concurrent.ExecutorService;
  39 import java.util.concurrent.Executors;
  40 import java.util.concurrent.Future;
  41 import java.util.concurrent.ScheduledExecutorService;
  42 import java.util.concurrent.ScheduledFuture;
  43 import java.util.concurrent.ThreadFactory;
  44 import java.util.concurrent.TimeUnit;
  45 import java.util.concurrent.atomic.AtomicBoolean;
  46 @Component("distribution-engine-cluster-health")
  47 public class DistributionEngineClusterHealth {
  48
  49     protected static String UEB_HEALTH_LOG_CONTEXT = "ueb.healthcheck";
  50
  51     //TODO use LoggerMetric instead
  52     private static final Logger healthLogger = Logger.getLogger(UEB_HEALTH_LOG_CONTEXT);
  53
  54     private static final String UEB_HEALTH_CHECK_STR = "uebHealthCheck";
  55
  56     boolean lastHealthState = false;
  57
  58     Object lockOject = new Object();
  59
  60     private long reconnectInterval = 5;
  61
  62     private long healthCheckReadTimeout = 20;
  63
  64     private static final Logger logger = Logger.getLogger(DistributionEngineClusterHealth.class.getName());
  65
  66     private List<String> uebServers = null;
  67
  68     private String publicApiKey = null;
  69
  70     public enum HealthCheckInfoResult {
  71
  72         OK(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.UP, null, ClusterStatusDescription.OK.getDescription())),
  73         UNAVAILABLE(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.DOWN, null, ClusterStatusDescription.UNAVAILABLE.getDescription())),
  74         NOT_CONFIGURED(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.DOWN, null, ClusterStatusDescription.NOT_CONFIGURED.getDescription())),
  75         DISABLED(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.DOWN, null, ClusterStatusDescription.DISABLED.getDescription())),
  76         UNKNOWN(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.UNKNOWN, null, ClusterStatusDescription.UNKNOWN.getDescription()));
  77
  78         private HealthCheckInfo healthCheckInfo;
  79
  80         HealthCheckInfoResult(HealthCheckInfo healthCheckInfo) {
  81             this.healthCheckInfo = healthCheckInfo;
  82         }
  83
  84         public HealthCheckInfo getHealthCheckInfo() {
  85             return healthCheckInfo;
  86         }
  87
  88     }
  89
  90     private HealthCheckInfo healthCheckInfo = HealthCheckInfoResult.UNKNOWN.getHealthCheckInfo();
  91
  92     private Map<String, AtomicBoolean> envNamePerStatus = null;
  93
  94     private ScheduledFuture<?> scheduledFuture = null;
  95
  96     ScheduledExecutorService healthCheckScheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
  97         @Override
  98         public Thread newThread(Runnable r) {
  99             return new Thread(r, "UEB-Health-Check-Task");
 100         }
 101     });
 102
 103     HealthCheckScheduledTask healthCheckScheduledTask = null;
 104
 105     public enum ClusterStatusDescription {
 106
 107         OK("OK"), UNAVAILABLE("U-EB cluster is not available"), NOT_CONFIGURED("U-EB cluster is not configured"), DISABLED("DE is disabled in configuration"), UNKNOWN("U-EB cluster is currently unknown (try again in few minutes)");
 108
 109         private String desc;
 110
 111         ClusterStatusDescription(String desc) {
 112             this.desc = desc;
 113         }
 114
 115         public String getDescription() {
 116             return desc;
 117         }
 118
 119     }
 120
 121     /**
 122      * Health Check Task Scheduler.
 123      *
 124      * It schedules a task which send a apiKey get query towards the UEB servers. In case a query to the first UEB server is failed, then a second query is sent to the next UEB server.
 125      *
 126      *
 127      * @author esofer
 128      *
 129      */
 130     public class HealthCheckScheduledTask implements Runnable {
 131
 132         List<UebHealthCheckCall> healthCheckCalls = new ArrayList<>();
 133
 134         public HealthCheckScheduledTask(List<String> uebServers) {
 135
 136             logger.debug("Create health check calls for servers {}", uebServers);
 137             if (uebServers != null) {
 138                 for (String server : uebServers) {
 139                     healthCheckCalls.add(new UebHealthCheckCall(server, publicApiKey));
 140                 }
 141             }
 142         }
 143
 144         @Override
 145         public void run() {
 146
 147             healthLogger.trace("Executing UEB Health Check Task - Start");
 148
 149             boolean healthStatus = verifyAtLeastOneEnvIsUp();
 150
 151             if (healthStatus) {
 152                 boolean queryUebStatus = queryUeb();
 153                 if (queryUebStatus == lastHealthState) {
 154                     return;
 155                 }
 156
 157                 synchronized (lockOject) {
 158                     if (queryUebStatus != lastHealthState) {
 159                         logger.trace("UEB Health State Changed to {}. Issuing alarm / recovery alarm...", healthStatus);
 160                         lastHealthState = queryUebStatus;
 161                         logAlarm(lastHealthState);
 162                         if (queryUebStatus) {
 163                             healthCheckInfo = HealthCheckInfoResult.OK.getHealthCheckInfo();
 164                         } else {
 165                             healthCheckInfo = HealthCheckInfoResult.UNAVAILABLE.getHealthCheckInfo();
 166                         }
 167                     }
 168                 }
 169             } else {
 170                 healthLogger.trace("Not all UEB Environments are up");
 171             }
 172
 173         }
 174
 175         /**
 176          * verify that at least one environment is up.
 177          *
 178          */
 179         private boolean verifyAtLeastOneEnvIsUp() {
 180
 181             boolean healthStatus = false;
 182
 183             if (envNamePerStatus != null) {
 184                 Collection<AtomicBoolean> values = envNamePerStatus.values();
 185                 if (values != null) {
 186                     for (AtomicBoolean status : values) {
 187                         if (status.get()) {
 188                             healthStatus = true;
 189                             break;
 190                         }
 191                     }
 192                 }
 193             }
 194
 195             return healthStatus;
 196         }
 197
 198         /**
 199          * executor for the query itself
 200          */
 201         ExecutorService healthCheckExecutor = Executors.newSingleThreadExecutor(new ThreadFactory() {
 202             @Override
 203             public Thread newThread(Runnable r) {
 204                 return new Thread(r, "UEB-Health-Check-Thread");
 205             }
 206         });
 207
 208         /**
 209          * go all UEB servers and send a get apiKeys query. In case a query is succeed, no query is sent to the rest of UEB servers.
 210          *
 211          *
 212          * @return
 213          */
 214         private boolean queryUeb() {
 215
 216             Boolean result = false;
 217             int retryNumber = 1;
 218             for (UebHealthCheckCall healthCheckCall : healthCheckCalls) {
 219                 try {
 220
 221                     healthLogger.debug("Before running Health Check retry query number {} towards UEB server {}", retryNumber, healthCheckCall.getServer());
 222
 223                     Future<Boolean> future = healthCheckExecutor.submit(healthCheckCall);
 224                     result = future.get(healthCheckReadTimeout, TimeUnit.SECONDS);
 225
 226                     healthLogger.debug("After running Health Check retry query number {} towards UEB server {}. Result is {}", retryNumber, healthCheckCall.getServer(), result);
 227
 228                     if (result != null && result.booleanValue()) {
 229                         break;
 230                     }
 231
 232                 } catch (Exception e) {
 233                     String message = e.getMessage();
 234                     if (message == null) {
 235                         message = e.getClass().getName();
 236                     }
 237                     healthLogger.debug("Error occured during running Health Check retry query towards UEB server {}. Result is {}", healthCheckCall.getServer(), message);
 238                     healthLogger.trace("Error occured during running Health Check retry query towards UEB server {}. Result is {}", healthCheckCall.getServer(), message, e);
 239                 }
 240                 retryNumber++;
 241
 242             }
 243
 244             return result;
 245
 246         }
 247
 248         public List<UebHealthCheckCall> getHealthCheckCalls() {
 249             return healthCheckCalls;
 250         }
 251
 252     }
 253
 254     @PostConstruct
 255     protected void init() {
 256
 257         logger.trace("Enter init method of DistributionEngineClusterHealth");
 258
 259         Long reconnectIntervalConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getUebHealthCheckReconnectIntervalInSeconds();
 260         if (reconnectIntervalConfig != null) {
 261             reconnectInterval = reconnectIntervalConfig.longValue();
 262         }
 263         Long healthCheckReadTimeoutConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getUebHealthCheckReadTimeout();
 264         if (healthCheckReadTimeoutConfig != null) {
 265             healthCheckReadTimeout = healthCheckReadTimeoutConfig.longValue();
 266         }
 267
 268         DistributionEngineConfiguration distributionEngineConfiguration = ConfigurationManager.getConfigurationManager().getDistributionEngineConfiguration();
 269
 270         this.uebServers = distributionEngineConfiguration.getUebServers();
 271         this.publicApiKey = distributionEngineConfiguration.getUebPublicKey();
 272
 273         this.healthCheckScheduledTask = new HealthCheckScheduledTask(this.uebServers);
 274
 275         logger.trace("Exit init method of DistributionEngineClusterHealth");
 276
 277     }
 278
 279     @PreDestroy
 280     protected void destroy() {
 281
 282         if (scheduledFuture != null) {
 283             scheduledFuture.cancel(true);
 284             scheduledFuture = null;
 285         }
 286
 287         if (healthCheckScheduler != null) {
 288             healthCheckScheduler.shutdown();
 289         }
 290
 291     }
 292
 293     /**
 294      * Start health check task.
 295      *
 296      * @param envNamePerStatus
 297      * @param startTask
 298      */
 299     public void startHealthCheckTask(Map<String, AtomicBoolean> envNamePerStatus, boolean startTask) {
 300         this.envNamePerStatus = envNamePerStatus;
 301
 302         if (startTask && this.scheduledFuture == null) {
 303             this.scheduledFuture = this.healthCheckScheduler.scheduleAtFixedRate(healthCheckScheduledTask, 0, reconnectInterval, TimeUnit.SECONDS);
 304         }
 305     }
 306
 307     public void startHealthCheckTask(Map<String, AtomicBoolean> envNamePerStatus) {
 308         startHealthCheckTask(envNamePerStatus, true);
 309     }
 310
 311     private void logAlarm(boolean lastHealthState) {
 312         if (lastHealthState) {
 313             BeEcompErrorManager.getInstance().logBeHealthCheckUebClusterRecovery(UEB_HEALTH_CHECK_STR);
 314         } else {
 315             BeEcompErrorManager.getInstance().logBeHealthCheckUebClusterError(UEB_HEALTH_CHECK_STR);
 316         }
 317     }
 318
 319     public HealthCheckInfo getHealthCheckInfo() {
 320         return healthCheckInfo;
 321     }
 322
 323     /**
 324      * change the health check to DISABLE
 325      */
 326     public void setHealthCheckUebIsDisabled() {
 327         healthCheckInfo = HealthCheckInfoResult.DISABLED.getHealthCheckInfo();
 328     }
 329
 330     /**
 331      * change the health check to NOT CONFGIURED
 332      */
 333     public void setHealthCheckUebConfigurationError() {
 334         healthCheckInfo = HealthCheckInfoResult.NOT_CONFIGURED.getHealthCheckInfo();
 335     }
 336
 337     public void setHealthCheckOkAndReportInCaseLastStateIsDown() {
 338
 339         if (lastHealthState) {
 340             return;
 341         }
 342         synchronized (lockOject) {
 343             if (!lastHealthState) {
 344                 logger.debug("Going to update health check state to available");
 345                 lastHealthState = true;
 346                 healthCheckInfo = HealthCheckInfoResult.OK.getHealthCheckInfo();
 347                 logAlarm(lastHealthState);
 348             }
 349         }
 350
 351     }
 352
 353 }