catalog-be/src/main/java/org/openecomp/sdc/be/components/distribution/engine/DistributionEngineClusterHealth.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.be.components.distribution.engine;
  22
  23 import org.openecomp.sdc.be.config.BeEcompErrorManager;
  24 import org.openecomp.sdc.be.config.ConfigurationManager;
  25 import org.openecomp.sdc.be.config.DistributionEngineConfiguration;
  26 import org.openecomp.sdc.common.api.Constants;
  27 import org.openecomp.sdc.common.api.HealthCheckInfo;
  28 import org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus;
  29 import org.openecomp.sdc.common.log.wrappers.Logger;
  30 import org.springframework.stereotype.Component;
  31
  32 import javax.annotation.PostConstruct;
  33 import javax.annotation.PreDestroy;
  34 import java.util.ArrayList;
  35 import java.util.Collection;
  36 import java.util.List;
  37 import java.util.Map;
  38 import java.util.concurrent.*;
  39 import java.util.concurrent.atomic.AtomicBoolean;
  40 @Component("distribution-engine-cluster-health")
  41 public class DistributionEngineClusterHealth {
  42
  43     protected static String UEB_HEALTH_LOG_CONTEXT = "ueb.healthcheck";
  44
  45     //TODO use LoggerMetric instead
  46     private static final Logger healthLogger = Logger.getLogger(UEB_HEALTH_LOG_CONTEXT);
  47
  48     private static final String UEB_HEALTH_CHECK_STR = "uebHealthCheck";
  49
  50     boolean lastHealthState = false;
  51
  52     Object lockOject = new Object();
  53
  54     private long reconnectInterval = 5;
  55
  56     private long healthCheckReadTimeout = 20;
  57
  58     private static final Logger logger = Logger.getLogger(DistributionEngineClusterHealth.class.getName());
  59
  60     private List<String> uebServers = null;
  61
  62     private String publicApiKey = null;
  63
  64     public enum HealthCheckInfoResult {
  65
  66         OK(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.UP, null, ClusterStatusDescription.OK.getDescription())),
  67         UNAVAILABLE(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.DOWN, null, ClusterStatusDescription.UNAVAILABLE.getDescription())),
  68         NOT_CONFIGURED(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.DOWN, null, ClusterStatusDescription.NOT_CONFIGURED.getDescription())),
  69         DISABLED(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.DOWN, null, ClusterStatusDescription.DISABLED.getDescription()));
  70
  71         private HealthCheckInfo healthCheckInfo;
  72
  73         HealthCheckInfoResult(HealthCheckInfo healthCheckInfo) {
  74             this.healthCheckInfo = healthCheckInfo;
  75         }
  76
  77         public HealthCheckInfo getHealthCheckInfo() {
  78             return healthCheckInfo;
  79         }
  80
  81     }
  82
  83     private HealthCheckInfo healthCheckInfo = HealthCheckInfoResult.UNAVAILABLE.getHealthCheckInfo();
  84
  85     private Map<String, AtomicBoolean> envNamePerStatus = null;
  86
  87     private ScheduledFuture<?> scheduledFuture = null;
  88
  89     ScheduledExecutorService healthCheckScheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
  90         @Override
  91         public Thread newThread(Runnable r) {
  92             return new Thread(r, "UEB-Health-Check-Task");
  93         }
  94     });
  95
  96     HealthCheckScheduledTask healthCheckScheduledTask = null;
  97
  98     public enum ClusterStatusDescription {
  99
 100         OK("OK"), UNAVAILABLE("U-EB cluster is not available"), NOT_CONFIGURED("U-EB cluster is not configured"), DISABLED("DE is disabled in configuration");
 101
 102         private String desc;
 103
 104         ClusterStatusDescription(String desc) {
 105             this.desc = desc;
 106         }
 107
 108         public String getDescription() {
 109             return desc;
 110         }
 111
 112     }
 113
 114     /**
 115      * Health Check Task Scheduler.
 116      *
 117      * It schedules a task which send a apiKey get query towards the UEB servers. In case a query to the first UEB server is failed, then a second query is sent to the next UEB server.
 118      *
 119      *
 120      * @author esofer
 121      *
 122      */
 123     public class HealthCheckScheduledTask implements Runnable {
 124
 125         List<UebHealthCheckCall> healthCheckCalls = new ArrayList<>();
 126
 127         public HealthCheckScheduledTask(List<String> uebServers) {
 128
 129             logger.debug("Create health check calls for servers {}", uebServers);
 130             if (uebServers != null) {
 131                 for (String server : uebServers) {
 132                     healthCheckCalls.add(new UebHealthCheckCall(server, publicApiKey));
 133                 }
 134             }
 135         }
 136
 137         @Override
 138         public void run() {
 139
 140             healthLogger.trace("Executing UEB Health Check Task - Start");
 141
 142             boolean healthStatus = verifyAtLeastOneEnvIsUp();
 143
 144             if (healthStatus) {
 145                 boolean queryUebStatus = queryUeb();
 146                 if (queryUebStatus == lastHealthState) {
 147                     return;
 148                 }
 149
 150                 synchronized (lockOject) {
 151                     if (queryUebStatus != lastHealthState) {
 152                         logger.trace("UEB Health State Changed to {}. Issuing alarm / recovery alarm...", healthStatus);
 153                         lastHealthState = queryUebStatus;
 154                         logAlarm(lastHealthState);
 155                         if (queryUebStatus) {
 156                             healthCheckInfo = HealthCheckInfoResult.OK.getHealthCheckInfo();
 157                         } else {
 158                             healthCheckInfo = HealthCheckInfoResult.UNAVAILABLE.getHealthCheckInfo();
 159                         }
 160                     }
 161                 }
 162             } else {
 163                 healthLogger.trace("Not all UEB Environments are up");
 164             }
 165
 166         }
 167
 168         /**
 169          * verify that at least one environment is up.
 170          *
 171          */
 172         private boolean verifyAtLeastOneEnvIsUp() {
 173
 174             boolean healthStatus = false;
 175
 176             if (envNamePerStatus != null) {
 177                 Collection<AtomicBoolean> values = envNamePerStatus.values();
 178                 if (values != null) {
 179                     for (AtomicBoolean status : values) {
 180                         if (status.get()) {
 181                             healthStatus = true;
 182                             break;
 183                         }
 184                     }
 185                 }
 186             }
 187
 188             return healthStatus;
 189         }
 190
 191         /**
 192          * executor for the query itself
 193          */
 194         ExecutorService healthCheckExecutor = Executors.newSingleThreadExecutor(new ThreadFactory() {
 195             @Override
 196             public Thread newThread(Runnable r) {
 197                 return new Thread(r, "UEB-Health-Check-Thread");
 198             }
 199         });
 200
 201         /**
 202          * go all UEB servers and send a get apiKeys query. In case a query is succeed, no query is sent to the rest of UEB servers.
 203          *
 204          *
 205          * @return
 206          */
 207         private boolean queryUeb() {
 208
 209             Boolean result = false;
 210             int retryNumber = 1;
 211             for (UebHealthCheckCall healthCheckCall : healthCheckCalls) {
 212                 try {
 213
 214                     healthLogger.debug("Before running Health Check retry query number {} towards UEB server {}", retryNumber, healthCheckCall.getServer());
 215
 216                     Future<Boolean> future = healthCheckExecutor.submit(healthCheckCall);
 217                     result = future.get(healthCheckReadTimeout, TimeUnit.SECONDS);
 218
 219                     healthLogger.debug("After running Health Check retry query number {} towards UEB server {}. Result is {}", retryNumber, healthCheckCall.getServer(), result);
 220
 221                     if (result != null && result.booleanValue()) {
 222                         break;
 223                     }
 224
 225                 } catch (Exception e) {
 226                     String message = e.getMessage();
 227                     if (message == null) {
 228                         message = e.getClass().getName();
 229                     }
 230                     healthLogger.debug("Error occured during running Health Check retry query towards UEB server {}. Result is {}", healthCheckCall.getServer(), message);
 231                     healthLogger.trace("Error occured during running Health Check retry query towards UEB server {}. Result is {}", healthCheckCall.getServer(), message, e);
 232                 }
 233                 retryNumber++;
 234
 235             }
 236
 237             return result;
 238
 239         }
 240
 241         public List<UebHealthCheckCall> getHealthCheckCalls() {
 242             return healthCheckCalls;
 243         }
 244
 245     }
 246
 247     @PostConstruct
 248     protected void init() {
 249
 250         logger.trace("Enter init method of DistributionEngineClusterHealth");
 251
 252         Long reconnectIntervalConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getUebHealthCheckReconnectIntervalInSeconds();
 253         if (reconnectIntervalConfig != null) {
 254             reconnectInterval = reconnectIntervalConfig.longValue();
 255         }
 256         Long healthCheckReadTimeoutConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getUebHealthCheckReadTimeout();
 257         if (healthCheckReadTimeoutConfig != null) {
 258             healthCheckReadTimeout = healthCheckReadTimeoutConfig.longValue();
 259         }
 260
 261         DistributionEngineConfiguration distributionEngineConfiguration = ConfigurationManager.getConfigurationManager().getDistributionEngineConfiguration();
 262
 263         this.uebServers = distributionEngineConfiguration.getUebServers();
 264         this.publicApiKey = distributionEngineConfiguration.getUebPublicKey();
 265
 266         this.healthCheckScheduledTask = new HealthCheckScheduledTask(this.uebServers);
 267
 268         logger.trace("Exit init method of DistributionEngineClusterHealth");
 269
 270     }
 271
 272     @PreDestroy
 273     protected void destroy() {
 274
 275         if (scheduledFuture != null) {
 276             scheduledFuture.cancel(true);
 277             scheduledFuture = null;
 278         }
 279
 280         if (healthCheckScheduler != null) {
 281             healthCheckScheduler.shutdown();
 282         }
 283
 284     }
 285
 286     /**
 287      * Start health check task.
 288      *
 289      * @param envNamePerStatus
 290      * @param startTask
 291      */
 292     public void startHealthCheckTask(Map<String, AtomicBoolean> envNamePerStatus, boolean startTask) {
 293         this.envNamePerStatus = envNamePerStatus;
 294
 295         if (startTask && this.scheduledFuture == null) {
 296             this.scheduledFuture = this.healthCheckScheduler.scheduleAtFixedRate(healthCheckScheduledTask, 0, reconnectInterval, TimeUnit.SECONDS);
 297         }
 298     }
 299
 300     public void startHealthCheckTask(Map<String, AtomicBoolean> envNamePerStatus) {
 301         startHealthCheckTask(envNamePerStatus, true);
 302     }
 303
 304     private void logAlarm(boolean lastHealthState) {
 305         if (lastHealthState) {
 306             BeEcompErrorManager.getInstance().logBeHealthCheckUebClusterRecovery(UEB_HEALTH_CHECK_STR);
 307         } else {
 308             BeEcompErrorManager.getInstance().logBeHealthCheckUebClusterError(UEB_HEALTH_CHECK_STR);
 309         }
 310     }
 311
 312     public HealthCheckInfo getHealthCheckInfo() {
 313         return healthCheckInfo;
 314     }
 315
 316     /**
 317      * change the health check to DISABLE
 318      */
 319     public void setHealthCheckUebIsDisabled() {
 320         healthCheckInfo = HealthCheckInfoResult.DISABLED.getHealthCheckInfo();
 321     }
 322
 323     /**
 324      * change the health check to NOT CONFGIURED
 325      */
 326     public void setHealthCheckUebConfigurationError() {
 327         healthCheckInfo = HealthCheckInfoResult.NOT_CONFIGURED.getHealthCheckInfo();
 328     }
 329
 330     public void setHealthCheckOkAndReportInCaseLastStateIsDown() {
 331
 332         if (lastHealthState) {
 333             return;
 334         }
 335         synchronized (lockOject) {
 336             if (!lastHealthState) {
 337                 logger.debug("Going to update health check state to available");
 338                 lastHealthState = true;
 339                 healthCheckInfo = HealthCheckInfoResult.OK.getHealthCheckInfo();
 340                 logAlarm(lastHealthState);
 341             }
 342         }
 343
 344     }
 345
 346 }