gerrit.onap Code Review - sdc.git/blob

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20 package org.openecomp.sdc.be.components.distribution.engine;
  21
  22 import java.util.ArrayList;
  23 import java.util.Collection;
  24 import java.util.List;
  25 import java.util.Map;
  26 import java.util.concurrent.ExecutorService;
  27 import java.util.concurrent.Executors;
  28 import java.util.concurrent.Future;
  29 import java.util.concurrent.ScheduledExecutorService;
  30 import java.util.concurrent.ScheduledFuture;
  31 import java.util.concurrent.ThreadFactory;
  32 import java.util.concurrent.TimeUnit;
  33 import java.util.concurrent.atomic.AtomicBoolean;
  34 import javax.annotation.PostConstruct;
  35 import javax.annotation.PreDestroy;
  36 import org.openecomp.sdc.be.config.BeEcompErrorManager;
  37 import org.openecomp.sdc.be.config.ConfigurationManager;
  38 import org.openecomp.sdc.be.config.DistributionEngineConfiguration;
  39 import org.openecomp.sdc.common.api.Constants;
  40 import org.openecomp.sdc.common.api.HealthCheckInfo;
  41 import org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus;
  42 import org.openecomp.sdc.common.log.wrappers.Logger;
  43 import org.springframework.stereotype.Component;
  44
  45 @Component("distribution-engine-cluster-health")
  46 public class DistributionEngineClusterHealth {
  47
  48     private static final String UEB_HEALTH_CHECK_STR = "uebHealthCheck";
  49     private static final Logger logger = Logger.getLogger(DistributionEngineClusterHealth.class.getName());
  50     protected static String UEB_HEALTH_LOG_CONTEXT = "ueb.healthcheck";
  51     //TODO use LoggerMetric instead
  52     private static final Logger healthLogger = Logger.getLogger(UEB_HEALTH_LOG_CONTEXT);
  53     boolean lastHealthState = false;
  54     Object lockOject = new Object();
  55     ScheduledExecutorService healthCheckScheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
  56         @Override
  57         public Thread newThread(Runnable r) {
  58             return new Thread(r, "UEB-Health-Check-Task");
  59         }
  60     });
  61     HealthCheckScheduledTask healthCheckScheduledTask = null;
  62     private long reconnectInterval = 5;
  63     private long healthCheckReadTimeout = 20;
  64     private List<String> uebServers = null;
  65     private String publicApiKey = null;
  66     private HealthCheckInfo healthCheckInfo = HealthCheckInfoResult.UNKNOWN.getHealthCheckInfo();
  67     private Map<String, AtomicBoolean> envNamePerStatus = null;
  68     private ScheduledFuture<?> scheduledFuture = null;
  69
  70     @PostConstruct
  71     protected void init() {
  72         logger.trace("Enter init method of DistributionEngineClusterHealth");
  73         Long reconnectIntervalConfig = ConfigurationManager.getConfigurationManager().getConfiguration()
  74             .getUebHealthCheckReconnectIntervalInSeconds();
  75         if (reconnectIntervalConfig != null) {
  76             reconnectInterval = reconnectIntervalConfig.longValue();
  77         }
  78         Long healthCheckReadTimeoutConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getUebHealthCheckReadTimeout();
  79         if (healthCheckReadTimeoutConfig != null) {
  80             healthCheckReadTimeout = healthCheckReadTimeoutConfig.longValue();
  81         }
  82         DistributionEngineConfiguration distributionEngineConfiguration = ConfigurationManager.getConfigurationManager()
  83             .getDistributionEngineConfiguration();
  84         this.uebServers = distributionEngineConfiguration.getUebServers();
  85         this.publicApiKey = distributionEngineConfiguration.getUebPublicKey();
  86         this.healthCheckScheduledTask = new HealthCheckScheduledTask(this.uebServers);
  87         logger.trace("Exit init method of DistributionEngineClusterHealth");
  88     }
  89
  90     @PreDestroy
  91     protected void destroy() {
  92         if (scheduledFuture != null) {
  93             scheduledFuture.cancel(true);
  94             scheduledFuture = null;
  95         }
  96         if (healthCheckScheduler != null) {
  97             healthCheckScheduler.shutdown();
  98         }
  99     }
 100
 101     /**
 102      * Start health check task.
 103      *
 104      * @param envNamePerStatus
 105      * @param startTask
 106      */
 107     public void startHealthCheckTask(Map<String, AtomicBoolean> envNamePerStatus, boolean startTask) {
 108         this.envNamePerStatus = envNamePerStatus;
 109         if (startTask && this.scheduledFuture == null) {
 110             this.scheduledFuture = this.healthCheckScheduler.scheduleAtFixedRate(healthCheckScheduledTask, 0, reconnectInterval, TimeUnit.SECONDS);
 111         }
 112     }
 113
 114     public void startHealthCheckTask(Map<String, AtomicBoolean> envNamePerStatus) {
 115         startHealthCheckTask(envNamePerStatus, true);
 116     }
 117
 118     private void logAlarm(boolean lastHealthState) {
 119         if (lastHealthState) {
 120             BeEcompErrorManager.getInstance().logBeHealthCheckUebClusterRecovery(UEB_HEALTH_CHECK_STR);
 121         } else {
 122             BeEcompErrorManager.getInstance().logBeHealthCheckUebClusterError(UEB_HEALTH_CHECK_STR);
 123         }
 124     }
 125
 126     public HealthCheckInfo getHealthCheckInfo() {
 127         return healthCheckInfo;
 128     }
 129
 130     /**
 131      * change the health check to DISABLE
 132      */
 133     public void setHealthCheckUebIsDisabled() {
 134         healthCheckInfo = HealthCheckInfoResult.DISABLED.getHealthCheckInfo();
 135     }
 136
 137     /**
 138      * change the health check to NOT CONFGIURED
 139      */
 140     public void setHealthCheckUebConfigurationError() {
 141         healthCheckInfo = HealthCheckInfoResult.NOT_CONFIGURED.getHealthCheckInfo();
 142     }
 143
 144     public void setHealthCheckOkAndReportInCaseLastStateIsDown() {
 145         if (lastHealthState) {
 146             return;
 147         }
 148         synchronized (lockOject) {
 149             if (!lastHealthState) {
 150                 logger.debug("Going to update health check state to available");
 151                 lastHealthState = true;
 152                 healthCheckInfo = HealthCheckInfoResult.OK.getHealthCheckInfo();
 153                 logAlarm(lastHealthState);
 154             }
 155         }
 156     }
 157
 158     public enum HealthCheckInfoResult {
 159         OK(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.UP, null,
 160             ClusterStatusDescription.OK.getDescription())), UNAVAILABLE(
 161             new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.DOWN, null,
 162                 ClusterStatusDescription.UNAVAILABLE.getDescription())), NOT_CONFIGURED(
 163             new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.DOWN, null,
 164                 ClusterStatusDescription.NOT_CONFIGURED.getDescription())), DISABLED(
 165             new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.DOWN, null,
 166                 ClusterStatusDescription.DISABLED.getDescription())), UNKNOWN(
 167             new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.UNKNOWN, null,
 168                 ClusterStatusDescription.UNKNOWN.getDescription()));
 169         private HealthCheckInfo healthCheckInfo;
 170
 171         HealthCheckInfoResult(HealthCheckInfo healthCheckInfo) {
 172             this.healthCheckInfo = healthCheckInfo;
 173         }
 174
 175         public HealthCheckInfo getHealthCheckInfo() {
 176             return healthCheckInfo;
 177         }
 178     }
 179
 180     public enum ClusterStatusDescription {
 181         OK("OK"), UNAVAILABLE("U-EB cluster is not available"), NOT_CONFIGURED("U-EB cluster is not configured"), DISABLED(
 182             "DE is disabled in configuration"), UNKNOWN("U-EB cluster is currently unknown (try again in few minutes)");
 183         private String desc;
 184
 185         ClusterStatusDescription(String desc) {
 186             this.desc = desc;
 187         }
 188
 189         public String getDescription() {
 190             return desc;
 191         }
 192     }
 193
 194     /**
 195      * Health Check Task Scheduler.
 196      * <p>
 197      * It schedules a task which send a apiKey get query towards the UEB servers. In case a query to the first UEB server is failed, then a second
 198      * query is sent to the next UEB server.
 199      *
 200      * @author esofer
 201      */
 202     public class HealthCheckScheduledTask implements Runnable {
 203
 204         List<UebHealthCheckCall> healthCheckCalls = new ArrayList<>();
 205         /**
 206          * executor for the query itself
 207          */
 208         ExecutorService healthCheckExecutor = Executors.newSingleThreadExecutor(new ThreadFactory() {
 209             @Override
 210             public Thread newThread(Runnable r) {
 211                 return new Thread(r, "UEB-Health-Check-Thread");
 212             }
 213         });
 214
 215         public HealthCheckScheduledTask(List<String> uebServers) {
 216             logger.debug("Create health check calls for servers {}", uebServers);
 217             if (uebServers != null) {
 218                 for (String server : uebServers) {
 219                     healthCheckCalls.add(new UebHealthCheckCall(server, publicApiKey));
 220                 }
 221             }
 222         }
 223
 224         @Override
 225         public void run() {
 226             healthLogger.trace("Executing UEB Health Check Task - Start");
 227             boolean healthStatus = verifyAtLeastOneEnvIsUp();
 228             if (healthStatus) {
 229                 boolean queryUebStatus = queryUeb();
 230                 if (queryUebStatus == lastHealthState) {
 231                     return;
 232                 }
 233                 synchronized (lockOject) {
 234                     if (queryUebStatus != lastHealthState) {
 235                         logger.trace("UEB Health State Changed to {}. Issuing alarm / recovery alarm...", healthStatus);
 236                         lastHealthState = queryUebStatus;
 237                         logAlarm(lastHealthState);
 238                         if (queryUebStatus) {
 239                             healthCheckInfo = HealthCheckInfoResult.OK.getHealthCheckInfo();
 240                         } else {
 241                             healthCheckInfo = HealthCheckInfoResult.UNAVAILABLE.getHealthCheckInfo();
 242                         }
 243                     }
 244                 }
 245             } else {
 246                 healthLogger.trace("Not all UEB Environments are up");
 247             }
 248         }
 249
 250         /**
 251          * verify that at least one environment is up.
 252          */
 253         private boolean verifyAtLeastOneEnvIsUp() {
 254             boolean healthStatus = false;
 255             if (envNamePerStatus != null) {
 256                 Collection<AtomicBoolean> values = envNamePerStatus.values();
 257                 if (values != null) {
 258                     for (AtomicBoolean status : values) {
 259                         if (status.get()) {
 260                             healthStatus = true;
 261                             break;
 262                         }
 263                     }
 264                 }
 265             }
 266             return healthStatus;
 267         }
 268
 269         /**
 270          * go all UEB servers and send a get apiKeys query. In case a query is succeed, no query is sent to the rest of UEB servers.
 271          *
 272          * @return
 273          */
 274         private boolean queryUeb() {
 275             Boolean result = false;
 276             int retryNumber = 1;
 277             for (UebHealthCheckCall healthCheckCall : healthCheckCalls) {
 278                 try {
 279                     healthLogger
 280                         .debug("Before running Health Check retry query number {} towards UEB server {}", retryNumber, healthCheckCall.getServer());
 281                     Future<Boolean> future = healthCheckExecutor.submit(healthCheckCall);
 282                     result = future.get(healthCheckReadTimeout, TimeUnit.SECONDS);
 283                     healthLogger.debug("After running Health Check retry query number {} towards UEB server {}. Result is {}", retryNumber,
 284                         healthCheckCall.getServer(), result);
 285                     if (result != null && result.booleanValue()) {
 286                         break;
 287                     }
 288                 } catch (Exception e) {
 289                     String message = e.getMessage();
 290                     if (message == null) {
 291                         message = e.getClass().getName();
 292                     }
 293                     healthLogger.debug("Error occured during running Health Check retry query towards UEB server {}. Result is {}",
 294                         healthCheckCall.getServer(), message);
 295                     healthLogger.trace("Error occured during running Health Check retry query towards UEB server {}. Result is {}",
 296                         healthCheckCall.getServer(), message, e);
 297                 }
 298                 retryNumber++;
 299             }
 300             return result;
 301         }
 302
 303         public List<UebHealthCheckCall> getHealthCheckCalls() {
 304             return healthCheckCalls;
 305         }
 306     }
 307 }