gerrit.onap Code Review - sdc.git/blob

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20 package org.openecomp.sdc.be.components.distribution.engine;
  21
  22 import java.util.ArrayList;
  23 import java.util.Collection;
  24 import java.util.List;
  25 import java.util.Map;
  26 import java.util.concurrent.ExecutorService;
  27 import java.util.concurrent.Executors;
  28 import java.util.concurrent.Future;
  29 import java.util.concurrent.ScheduledExecutorService;
  30 import java.util.concurrent.ScheduledFuture;
  31 import java.util.concurrent.ThreadFactory;
  32 import java.util.concurrent.TimeUnit;
  33 import java.util.concurrent.atomic.AtomicBoolean;
  34 import javax.annotation.PreDestroy;
  35 import lombok.AllArgsConstructor;
  36 import lombok.Getter;
  37 import org.openecomp.sdc.be.config.BeEcompErrorManager;
  38 import org.openecomp.sdc.be.config.ConfigurationManager;
  39 import org.openecomp.sdc.be.config.DistributionEngineConfiguration;
  40 import org.openecomp.sdc.common.api.Constants;
  41 import org.openecomp.sdc.common.api.HealthCheckInfo;
  42 import org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus;
  43 import org.openecomp.sdc.common.log.wrappers.Logger;
  44 import org.springframework.stereotype.Component;
  45
  46 @Component("distribution-engine-cluster-health")
  47 public class DistributionEngineClusterHealth {
  48
  49     private static final String UEB_HEALTH_CHECK_STR = "uebHealthCheck";
  50     private static final Logger logger = Logger.getLogger(DistributionEngineClusterHealth.class.getName());
  51     protected static final String UEB_HEALTH_LOG_CONTEXT = "ueb.healthcheck";
  52     //TODO use LoggerMetric instead
  53     private static final Logger healthLogger = Logger.getLogger(UEB_HEALTH_LOG_CONTEXT);
  54     boolean lastHealthState = false;
  55     Object lockOject = new Object();
  56     ScheduledExecutorService healthCheckScheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
  57         @Override
  58         public Thread newThread(Runnable r) {
  59             return new Thread(r, "UEB-Health-Check-Task");
  60         }
  61     });
  62     HealthCheckScheduledTask healthCheckScheduledTask = null;
  63     private long reconnectInterval = 5;
  64     private long healthCheckReadTimeout = 20;
  65     private List<String> uebServers = null;
  66     private String publicApiKey = null;
  67     private HealthCheckInfo healthCheckInfo = HealthCheckInfoResult.UNKNOWN.getHealthCheckInfo();
  68     private Map<String, AtomicBoolean> envNamePerStatus = null;
  69     private ScheduledFuture<?> scheduledFuture = null;
  70
  71     protected void init(final String publicApiKey) {
  72         logger.trace("Enter init method of DistributionEngineClusterHealth");
  73         Long reconnectIntervalConfig = ConfigurationManager.getConfigurationManager().getConfiguration()
  74             .getUebHealthCheckReconnectIntervalInSeconds();
  75         if (reconnectIntervalConfig != null) {
  76             reconnectInterval = reconnectIntervalConfig.longValue();
  77         }
  78         Long healthCheckReadTimeoutConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getUebHealthCheckReadTimeout();
  79         if (healthCheckReadTimeoutConfig != null) {
  80             healthCheckReadTimeout = healthCheckReadTimeoutConfig.longValue();
  81         }
  82         DistributionEngineConfiguration distributionEngineConfiguration = ConfigurationManager.getConfigurationManager()
  83             .getDistributionEngineConfiguration();
  84         this.uebServers = distributionEngineConfiguration.getUebServers();
  85         this.publicApiKey = publicApiKey;
  86         this.healthCheckScheduledTask = new HealthCheckScheduledTask(this.uebServers);
  87         logger.trace("Exit init method of DistributionEngineClusterHealth");
  88     }
  89
  90     @PreDestroy
  91     protected void destroy() {
  92         if (scheduledFuture != null) {
  93             scheduledFuture.cancel(true);
  94             scheduledFuture = null;
  95         }
  96         if (healthCheckScheduler != null) {
  97             healthCheckScheduler.shutdown();
  98         }
  99     }
 100
 101     /**
 102      * Start health check task.
 103      *
 104      * @param envNamePerStatus
 105      * @param startTask
 106      */
 107     public void startHealthCheckTask(Map<String, AtomicBoolean> envNamePerStatus, boolean startTask) {
 108         this.envNamePerStatus = envNamePerStatus;
 109         if (startTask && this.scheduledFuture == null) {
 110             this.scheduledFuture = this.healthCheckScheduler.scheduleAtFixedRate(healthCheckScheduledTask, 0, reconnectInterval, TimeUnit.SECONDS);
 111         }
 112     }
 113
 114     public void startHealthCheckTask(Map<String, AtomicBoolean> envNamePerStatus) {
 115         startHealthCheckTask(envNamePerStatus, true);
 116     }
 117
 118     private void logAlarm(boolean lastHealthState) {
 119         if (lastHealthState) {
 120             BeEcompErrorManager.getInstance().logBeHealthCheckUebClusterRecovery(UEB_HEALTH_CHECK_STR);
 121         } else {
 122             BeEcompErrorManager.getInstance().logBeHealthCheckUebClusterError(UEB_HEALTH_CHECK_STR);
 123         }
 124     }
 125
 126     public HealthCheckInfo getHealthCheckInfo() {
 127         return healthCheckInfo;
 128     }
 129
 130     /**
 131      * change the health check to DISABLE
 132      */
 133     public void setHealthCheckUebIsDisabled() {
 134         healthCheckInfo = HealthCheckInfoResult.DISABLED.getHealthCheckInfo();
 135     }
 136
 137     /**
 138      * change the health check to NOT CONFGIURED
 139      */
 140     public void setHealthCheckUebConfigurationError() {
 141         healthCheckInfo = HealthCheckInfoResult.NOT_CONFIGURED.getHealthCheckInfo();
 142     }
 143
 144     public void setHealthCheckOkAndReportInCaseLastStateIsDown() {
 145         if (lastHealthState) {
 146             return;
 147         }
 148         synchronized (lockOject) {
 149             if (!lastHealthState) {
 150                 logger.debug("Going to update health check state to available");
 151                 lastHealthState = true;
 152                 healthCheckInfo = HealthCheckInfoResult.OK.getHealthCheckInfo();
 153                 logAlarm(lastHealthState);
 154             }
 155         }
 156     }
 157
 158     @AllArgsConstructor
 159     @Getter
 160     private enum HealthCheckInfoResult {
 161         // @formatter:off
 162         OK              (new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.UP, null, ClusterStatusDescription.OK.getDescription())),
 163         UNAVAILABLE     (new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.DOWN, null, ClusterStatusDescription.UNAVAILABLE.getDescription())),
 164         NOT_CONFIGURED  (new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.DOWN, null, ClusterStatusDescription.NOT_CONFIGURED.getDescription())),
 165         DISABLED        (new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.DOWN, null, ClusterStatusDescription.DISABLED.getDescription())),
 166         UNKNOWN         (new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.UNKNOWN, null, ClusterStatusDescription.UNKNOWN.getDescription()));
 167         // @formatter:on
 168         private final HealthCheckInfo healthCheckInfo;
 169     }
 170
 171     @AllArgsConstructor
 172     @Getter
 173     private enum ClusterStatusDescription {
 174         OK("OK"),
 175         UNAVAILABLE("U-EB cluster is not available"),
 176         NOT_CONFIGURED("U-EB cluster is not configured"),
 177         DISABLED("DE is disabled in configuration"),
 178         UNKNOWN("U-EB cluster is currently unknown (try again in few minutes)");
 179         private final String description;
 180     }
 181
 182     /**
 183      * Health Check Task Scheduler.
 184      * <p>
 185      * It schedules a task which send a apiKey get query towards the UEB servers. In case a query to the first UEB server is failed, then a second
 186      * query is sent to the next UEB server.
 187      *
 188      * @author esofer
 189      */
 190     public class HealthCheckScheduledTask implements Runnable {
 191
 192         @Getter
 193         List<UebHealthCheckCall> healthCheckCalls = new ArrayList<>();
 194         /**
 195          * executor for the query itself
 196          */
 197         private final ExecutorService healthCheckExecutor = Executors.newSingleThreadExecutor(new ThreadFactory() {
 198             @Override
 199             public Thread newThread(Runnable r) {
 200                 return new Thread(r, "UEB-Health-Check-Thread");
 201             }
 202         });
 203
 204         public HealthCheckScheduledTask(List<String> localUebServers) {
 205             logger.debug("Create health check calls for servers {}", localUebServers);
 206             if (localUebServers != null) {
 207                 for (String server : localUebServers) {
 208                     healthCheckCalls.add(new UebHealthCheckCall(server, publicApiKey));
 209                 }
 210             }
 211         }
 212
 213         @Override
 214         public void run() {
 215             healthLogger.trace("Executing UEB Health Check Task - Start");
 216             boolean healthStatus = verifyAtLeastOneEnvIsUp();
 217             if (healthStatus) {
 218                 boolean queryUebStatus = queryUeb();
 219                 if (queryUebStatus == lastHealthState) {
 220                     return;
 221                 }
 222                 synchronized (lockOject) {
 223                     if (queryUebStatus != lastHealthState) {
 224                         logger.trace("UEB Health State Changed to {}. Issuing alarm / recovery alarm...", healthStatus);
 225                         lastHealthState = queryUebStatus;
 226                         logAlarm(lastHealthState);
 227                         if (queryUebStatus) {
 228                             healthCheckInfo = HealthCheckInfoResult.OK.getHealthCheckInfo();
 229                         } else {
 230                             healthCheckInfo = HealthCheckInfoResult.UNAVAILABLE.getHealthCheckInfo();
 231                         }
 232                     }
 233                 }
 234             } else {
 235                 healthLogger.trace("Not all UEB Environments are up");
 236             }
 237         }
 238
 239         /**
 240          * verify that at least one environment is up.
 241          */
 242         private boolean verifyAtLeastOneEnvIsUp() {
 243             boolean healthStatus = false;
 244             if (envNamePerStatus != null) {
 245                 Collection<AtomicBoolean> values = envNamePerStatus.values();
 246                 if (values != null) {
 247                     for (AtomicBoolean status : values) {
 248                         if (status.get()) {
 249                             healthStatus = true;
 250                             break;
 251                         }
 252                     }
 253                 }
 254             }
 255             return healthStatus;
 256         }
 257
 258         /**
 259          * go all UEB servers and send a get apiKeys query. In case a query is succeed, no query is sent to the rest of UEB servers.
 260          *
 261          * @return
 262          */
 263         private boolean queryUeb() {
 264             Boolean result = false;
 265             int retryNumber = 1;
 266             for (UebHealthCheckCall healthCheckCall : healthCheckCalls) {
 267                 try {
 268                     healthLogger
 269                         .debug("Before running Health Check retry query number {} towards UEB server {}", retryNumber, healthCheckCall.getServer());
 270                     Future<Boolean> future = healthCheckExecutor.submit(healthCheckCall);
 271                     result = future.get(healthCheckReadTimeout, TimeUnit.SECONDS);
 272                     healthLogger.debug("After running Health Check retry query number {} towards UEB server {}. Result is {}", retryNumber,
 273                         healthCheckCall.getServer(), result);
 274                     if (result != null && result.booleanValue()) {
 275                         break;
 276                     }
 277                 } catch (Exception e) {
 278                     String message = e.getMessage();
 279                     if (message == null) {
 280                         message = e.getClass().getName();
 281                     }
 282                     healthLogger.debug("Error occured during running Health Check retry query towards UEB server {}. Result is {}",
 283                         healthCheckCall.getServer(), message);
 284                     healthLogger.trace("Error occured during running Health Check retry query towards UEB server {}. Result is {}",
 285                         healthCheckCall.getServer(), message, e);
 286                 }
 287                 retryNumber++;
 288             }
 289             return result;
 290         }
 291
 292     }
 293 }