gerrit.onap Code Review - sdc.git/blob

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20 package org.openecomp.sdc.be.components.distribution.engine;
  21
  22 import java.util.ArrayList;
  23 import java.util.Collection;
  24 import java.util.List;
  25 import java.util.Map;
  26 import java.util.concurrent.ExecutorService;
  27 import java.util.concurrent.Executors;
  28 import java.util.concurrent.Future;
  29 import java.util.concurrent.ScheduledExecutorService;
  30 import java.util.concurrent.ScheduledFuture;
  31 import java.util.concurrent.ThreadFactory;
  32 import java.util.concurrent.TimeUnit;
  33 import java.util.concurrent.atomic.AtomicBoolean;
  34 import javax.annotation.PreDestroy;
  35 import org.openecomp.sdc.be.config.BeEcompErrorManager;
  36 import org.openecomp.sdc.be.config.ConfigurationManager;
  37 import org.openecomp.sdc.be.config.DistributionEngineConfiguration;
  38 import org.openecomp.sdc.common.api.Constants;
  39 import org.openecomp.sdc.common.api.HealthCheckInfo;
  40 import org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus;
  41 import org.openecomp.sdc.common.log.wrappers.Logger;
  42 import org.springframework.stereotype.Component;
  43
  44 @Component("distribution-engine-cluster-health")
  45 public class DistributionEngineClusterHealth {
  46
  47     private static final String UEB_HEALTH_CHECK_STR = "uebHealthCheck";
  48     private static final Logger logger = Logger.getLogger(DistributionEngineClusterHealth.class.getName());
  49     protected static final String UEB_HEALTH_LOG_CONTEXT = "ueb.healthcheck";
  50     //TODO use LoggerMetric instead
  51     private static final Logger healthLogger = Logger.getLogger(UEB_HEALTH_LOG_CONTEXT);
  52     boolean lastHealthState = false;
  53     Object lockOject = new Object();
  54     ScheduledExecutorService healthCheckScheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
  55         @Override
  56         public Thread newThread(Runnable r) {
  57             return new Thread(r, "UEB-Health-Check-Task");
  58         }
  59     });
  60     HealthCheckScheduledTask healthCheckScheduledTask = null;
  61     private long reconnectInterval = 5;
  62     private long healthCheckReadTimeout = 20;
  63     private List<String> uebServers = null;
  64     private String publicApiKey = null;
  65     private HealthCheckInfo healthCheckInfo = HealthCheckInfoResult.UNKNOWN.getHealthCheckInfo();
  66     private Map<String, AtomicBoolean> envNamePerStatus = null;
  67     private ScheduledFuture<?> scheduledFuture = null;
  68
  69     protected void init(final String publicApiKey) {
  70         logger.trace("Enter init method of DistributionEngineClusterHealth");
  71         Long reconnectIntervalConfig = ConfigurationManager.getConfigurationManager().getConfiguration()
  72             .getUebHealthCheckReconnectIntervalInSeconds();
  73         if (reconnectIntervalConfig != null) {
  74             reconnectInterval = reconnectIntervalConfig.longValue();
  75         }
  76         Long healthCheckReadTimeoutConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getUebHealthCheckReadTimeout();
  77         if (healthCheckReadTimeoutConfig != null) {
  78             healthCheckReadTimeout = healthCheckReadTimeoutConfig.longValue();
  79         }
  80         DistributionEngineConfiguration distributionEngineConfiguration = ConfigurationManager.getConfigurationManager()
  81             .getDistributionEngineConfiguration();
  82         this.uebServers = distributionEngineConfiguration.getUebServers();
  83         this.publicApiKey = publicApiKey;
  84         this.healthCheckScheduledTask = new HealthCheckScheduledTask(this.uebServers);
  85         logger.trace("Exit init method of DistributionEngineClusterHealth");
  86     }
  87
  88     @PreDestroy
  89     protected void destroy() {
  90         if (scheduledFuture != null) {
  91             scheduledFuture.cancel(true);
  92             scheduledFuture = null;
  93         }
  94         if (healthCheckScheduler != null) {
  95             healthCheckScheduler.shutdown();
  96         }
  97     }
  98
  99     /**
 100      * Start health check task.
 101      *
 102      * @param envNamePerStatus
 103      * @param startTask
 104      */
 105     public void startHealthCheckTask(Map<String, AtomicBoolean> envNamePerStatus, boolean startTask) {
 106         this.envNamePerStatus = envNamePerStatus;
 107         if (startTask && this.scheduledFuture == null) {
 108             this.scheduledFuture = this.healthCheckScheduler.scheduleAtFixedRate(healthCheckScheduledTask, 0, reconnectInterval, TimeUnit.SECONDS);
 109         }
 110     }
 111
 112     public void startHealthCheckTask(Map<String, AtomicBoolean> envNamePerStatus) {
 113         startHealthCheckTask(envNamePerStatus, true);
 114     }
 115
 116     private void logAlarm(boolean lastHealthState) {
 117         if (lastHealthState) {
 118             BeEcompErrorManager.getInstance().logBeHealthCheckUebClusterRecovery(UEB_HEALTH_CHECK_STR);
 119         } else {
 120             BeEcompErrorManager.getInstance().logBeHealthCheckUebClusterError(UEB_HEALTH_CHECK_STR);
 121         }
 122     }
 123
 124     public HealthCheckInfo getHealthCheckInfo() {
 125         return healthCheckInfo;
 126     }
 127
 128     /**
 129      * change the health check to DISABLE
 130      */
 131     public void setHealthCheckUebIsDisabled() {
 132         healthCheckInfo = HealthCheckInfoResult.DISABLED.getHealthCheckInfo();
 133     }
 134
 135     /**
 136      * change the health check to NOT CONFGIURED
 137      */
 138     public void setHealthCheckUebConfigurationError() {
 139         healthCheckInfo = HealthCheckInfoResult.NOT_CONFIGURED.getHealthCheckInfo();
 140     }
 141
 142     public void setHealthCheckOkAndReportInCaseLastStateIsDown() {
 143         if (lastHealthState) {
 144             return;
 145         }
 146         synchronized (lockOject) {
 147             if (!lastHealthState) {
 148                 logger.debug("Going to update health check state to available");
 149                 lastHealthState = true;
 150                 healthCheckInfo = HealthCheckInfoResult.OK.getHealthCheckInfo();
 151                 logAlarm(lastHealthState);
 152             }
 153         }
 154     }
 155
 156     public enum HealthCheckInfoResult {
 157         OK(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.UP, null,
 158             ClusterStatusDescription.OK.getDescription())), UNAVAILABLE(
 159             new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.DOWN, null,
 160                 ClusterStatusDescription.UNAVAILABLE.getDescription())), NOT_CONFIGURED(
 161             new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.DOWN, null,
 162                 ClusterStatusDescription.NOT_CONFIGURED.getDescription())), DISABLED(
 163             new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.DOWN, null,
 164                 ClusterStatusDescription.DISABLED.getDescription())), UNKNOWN(
 165             new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.UNKNOWN, null,
 166                 ClusterStatusDescription.UNKNOWN.getDescription()));
 167         private HealthCheckInfo healthCheckInfo;
 168
 169         HealthCheckInfoResult(HealthCheckInfo healthCheckInfo) {
 170             this.healthCheckInfo = healthCheckInfo;
 171         }
 172
 173         public HealthCheckInfo getHealthCheckInfo() {
 174             return healthCheckInfo;
 175         }
 176     }
 177
 178     public enum ClusterStatusDescription {
 179         OK("OK"), UNAVAILABLE("U-EB cluster is not available"), NOT_CONFIGURED("U-EB cluster is not configured"), DISABLED(
 180             "DE is disabled in configuration"), UNKNOWN("U-EB cluster is currently unknown (try again in few minutes)");
 181         private String desc;
 182
 183         ClusterStatusDescription(String desc) {
 184             this.desc = desc;
 185         }
 186
 187         public String getDescription() {
 188             return desc;
 189         }
 190     }
 191
 192     /**
 193      * Health Check Task Scheduler.
 194      * <p>
 195      * It schedules a task which send a apiKey get query towards the UEB servers. In case a query to the first UEB server is failed, then a second
 196      * query is sent to the next UEB server.
 197      *
 198      * @author esofer
 199      */
 200     public class HealthCheckScheduledTask implements Runnable {
 201
 202         List<UebHealthCheckCall> healthCheckCalls = new ArrayList<>();
 203         /**
 204          * executor for the query itself
 205          */
 206         ExecutorService healthCheckExecutor = Executors.newSingleThreadExecutor(new ThreadFactory() {
 207             @Override
 208             public Thread newThread(Runnable r) {
 209                 return new Thread(r, "UEB-Health-Check-Thread");
 210             }
 211         });
 212
 213         public HealthCheckScheduledTask(List<String> localUebServers) {
 214             logger.debug("Create health check calls for servers {}", localUebServers);
 215             if (localUebServers != null) {
 216                 for (String server : localUebServers) {
 217                     healthCheckCalls.add(new UebHealthCheckCall(server, publicApiKey));
 218                 }
 219             }
 220         }
 221
 222         @Override
 223         public void run() {
 224             healthLogger.trace("Executing UEB Health Check Task - Start");
 225             boolean healthStatus = verifyAtLeastOneEnvIsUp();
 226             if (healthStatus) {
 227                 boolean queryUebStatus = queryUeb();
 228                 if (queryUebStatus == lastHealthState) {
 229                     return;
 230                 }
 231                 synchronized (lockOject) {
 232                     if (queryUebStatus != lastHealthState) {
 233                         logger.trace("UEB Health State Changed to {}. Issuing alarm / recovery alarm...", healthStatus);
 234                         lastHealthState = queryUebStatus;
 235                         logAlarm(lastHealthState);
 236                         if (queryUebStatus) {
 237                             healthCheckInfo = HealthCheckInfoResult.OK.getHealthCheckInfo();
 238                         } else {
 239                             healthCheckInfo = HealthCheckInfoResult.UNAVAILABLE.getHealthCheckInfo();
 240                         }
 241                     }
 242                 }
 243             } else {
 244                 healthLogger.trace("Not all UEB Environments are up");
 245             }
 246         }
 247
 248         /**
 249          * verify that at least one environment is up.
 250          */
 251         private boolean verifyAtLeastOneEnvIsUp() {
 252             boolean healthStatus = false;
 253             if (envNamePerStatus != null) {
 254                 Collection<AtomicBoolean> values = envNamePerStatus.values();
 255                 if (values != null) {
 256                     for (AtomicBoolean status : values) {
 257                         if (status.get()) {
 258                             healthStatus = true;
 259                             break;
 260                         }
 261                     }
 262                 }
 263             }
 264             return healthStatus;
 265         }
 266
 267         /**
 268          * go all UEB servers and send a get apiKeys query. In case a query is succeed, no query is sent to the rest of UEB servers.
 269          *
 270          * @return
 271          */
 272         private boolean queryUeb() {
 273             Boolean result = false;
 274             int retryNumber = 1;
 275             for (UebHealthCheckCall healthCheckCall : healthCheckCalls) {
 276                 try {
 277                     healthLogger
 278                         .debug("Before running Health Check retry query number {} towards UEB server {}", retryNumber, healthCheckCall.getServer());
 279                     Future<Boolean> future = healthCheckExecutor.submit(healthCheckCall);
 280                     result = future.get(healthCheckReadTimeout, TimeUnit.SECONDS);
 281                     healthLogger.debug("After running Health Check retry query number {} towards UEB server {}. Result is {}", retryNumber,
 282                         healthCheckCall.getServer(), result);
 283                     if (result != null && result.booleanValue()) {
 284                         break;
 285                     }
 286                 } catch (Exception e) {
 287                     String message = e.getMessage();
 288                     if (message == null) {
 289                         message = e.getClass().getName();
 290                     }
 291                     healthLogger.debug("Error occured during running Health Check retry query towards UEB server {}. Result is {}",
 292                         healthCheckCall.getServer(), message);
 293                     healthLogger.trace("Error occured during running Health Check retry query towards UEB server {}. Result is {}",
 294                         healthCheckCall.getServer(), message, e);
 295                 }
 296                 retryNumber++;
 297             }
 298             return result;
 299         }
 300
 301         public List<UebHealthCheckCall> getHealthCheckCalls() {
 302             return healthCheckCalls;
 303         }
 304     }
 305 }