gerrit.onap Code Review - sdc.git/blob

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.be.components.distribution.engine;
  22
  23 import java.util.ArrayList;
  24 import java.util.Collection;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.concurrent.ExecutorService;
  28 import java.util.concurrent.Executors;
  29 import java.util.concurrent.Future;
  30 import java.util.concurrent.ScheduledExecutorService;
  31 import java.util.concurrent.ScheduledFuture;
  32 import java.util.concurrent.ThreadFactory;
  33 import java.util.concurrent.TimeUnit;
  34 import java.util.concurrent.atomic.AtomicBoolean;
  35
  36 import javax.annotation.PostConstruct;
  37 import javax.annotation.PreDestroy;
  38
  39 import org.openecomp.sdc.be.config.BeEcompErrorManager;
  40 import org.openecomp.sdc.be.config.ConfigurationManager;
  41 import org.openecomp.sdc.be.config.DistributionEngineConfiguration;
  42 import org.openecomp.sdc.common.api.Constants;
  43 import org.openecomp.sdc.common.api.HealthCheckInfo;
  44 import org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus;
  45 import org.openecomp.sdc.common.config.EcompErrorName;
  46 import org.slf4j.Logger;
  47 import org.slf4j.LoggerFactory;
  48 import org.springframework.stereotype.Component;
  49
  50 @Component("distribution-engine-cluster-health")
  51 public class DistributionEngineClusterHealth {
  52
  53         protected static String UEB_HEALTH_LOG_CONTEXT = "ueb.healthcheck";
  54
  55         private static Logger healthLogger = LoggerFactory.getLogger(UEB_HEALTH_LOG_CONTEXT);
  56
  57         private static final String UEB_HEALTH_CHECK_STR = "uebHealthCheck";
  58
  59         boolean lastHealthState = false;
  60
  61         Object lockOject = new Object();
  62
  63         private long reconnectInterval = 5;
  64
  65         private long healthCheckReadTimeout = 20;
  66
  67         private static Logger logger = LoggerFactory.getLogger(DistributionEngineClusterHealth.class.getName());
  68
  69         private List<String> uebServers = null;
  70
  71         private String publicApiKey = null;
  72
  73         public enum HealthCheckInfoResult {
  74
  75                 OK(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.UP, null, ClusterStatusDescription.OK.getDescription())),
  76                 UNAVAILABLE(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.DOWN, null, ClusterStatusDescription.UNAVAILABLE.getDescription())),
  77                 NOT_CONFIGURED(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.DOWN, null, ClusterStatusDescription.NOT_CONFIGURED.getDescription())),
  78                 DISABLED(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.DOWN, null, ClusterStatusDescription.DISABLED.getDescription()));
  79
  80                 private HealthCheckInfo healthCheckInfo;
  81
  82                 HealthCheckInfoResult(HealthCheckInfo healthCheckInfo) {
  83                         this.healthCheckInfo = healthCheckInfo;
  84                 }
  85
  86                 public HealthCheckInfo getHealthCheckInfo() {
  87                         return healthCheckInfo;
  88                 }
  89
  90         }
  91
  92         private HealthCheckInfo healthCheckInfo = HealthCheckInfoResult.UNAVAILABLE.getHealthCheckInfo();
  93
  94         private Map<String, AtomicBoolean> envNamePerStatus = null;
  95
  96         private ScheduledFuture<?> scheduledFuture = null;
  97
  98         ScheduledExecutorService healthCheckScheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
  99                 @Override
 100                 public Thread newThread(Runnable r) {
 101                         return new Thread(r, "UEB-Health-Check-Task");
 102                 }
 103         });
 104
 105         HealthCheckScheduledTask healthCheckScheduledTask = null;
 106
 107         public enum ClusterStatusDescription {
 108
 109                 OK("OK"), UNAVAILABLE("U-EB cluster is not available"), NOT_CONFIGURED("U-EB cluster is not configured"), DISABLED("DE is disabled in configuration");
 110
 111                 private String desc;
 112
 113                 ClusterStatusDescription(String desc) {
 114                         this.desc = desc;
 115                 }
 116
 117                 public String getDescription() {
 118                         return desc;
 119                 }
 120
 121         }
 122
 123         /**
 124          * Health Check Task Scheduler.
 125          *
 126          * It schedules a task which send a apiKey get query towards the UEB servers. In case a query to the first UEB server is failed, then a second query is sent to the next UEB server.
 127          *
 128          *
 129          * @author esofer
 130          *
 131          */
 132         public class HealthCheckScheduledTask implements Runnable {
 133
 134                 List<UebHealthCheckCall> healthCheckCalls = new ArrayList<>();
 135
 136                 public HealthCheckScheduledTask(List<String> uebServers) {
 137
 138                         logger.debug("Create health check calls for servers {}", uebServers);
 139                         if (uebServers != null) {
 140                                 for (String server : uebServers) {
 141                                         healthCheckCalls.add(new UebHealthCheckCall(server, publicApiKey));
 142                                 }
 143                         }
 144                 }
 145
 146                 @Override
 147                 public void run() {
 148
 149                         healthLogger.trace("Executing UEB Health Check Task - Start");
 150
 151                         boolean healthStatus = verifyAtLeastOneEnvIsUp();
 152
 153                         if (true == healthStatus) {
 154                                 boolean queryUebStatus = queryUeb();
 155                                 if (queryUebStatus == lastHealthState) {
 156                                         return;
 157                                 }
 158
 159                                 synchronized (lockOject) {
 160                                         if (queryUebStatus != lastHealthState) {
 161                                                 logger.trace("UEB Health State Changed to {}. Issuing alarm / recovery alarm...", healthStatus);
 162                                                 lastHealthState = queryUebStatus;
 163                                                 logAlarm(lastHealthState);
 164                                                 if (true == queryUebStatus) {
 165                                                         healthCheckInfo = HealthCheckInfoResult.OK.getHealthCheckInfo();
 166                                                 } else {
 167                                                         healthCheckInfo = HealthCheckInfoResult.UNAVAILABLE.getHealthCheckInfo();
 168                                                 }
 169                                         }
 170                                 }
 171                         } else {
 172                                 healthLogger.trace("Not all UEB Environments are up");
 173                         }
 174
 175                 }
 176
 177                 /**
 178                  * verify that at least one environment is up.
 179                  *
 180                  */
 181                 private boolean verifyAtLeastOneEnvIsUp() {
 182
 183                         boolean healthStatus = false;
 184
 185                         if (envNamePerStatus != null) {
 186                                 Collection<AtomicBoolean> values = envNamePerStatus.values();
 187                                 if (values != null) {
 188                                         for (AtomicBoolean status : values) {
 189                                                 if (true == status.get()) {
 190                                                         healthStatus = true;
 191                                                         break;
 192                                                 }
 193                                         }
 194                                 }
 195                         }
 196
 197                         return healthStatus;
 198                 }
 199
 200                 /**
 201                  * executor for the query itself
 202                  */
 203                 ExecutorService healthCheckExecutor = Executors.newSingleThreadExecutor(new ThreadFactory() {
 204                         @Override
 205                         public Thread newThread(Runnable r) {
 206                                 return new Thread(r, "UEB-Health-Check-Thread");
 207                         }
 208                 });
 209
 210                 /**
 211                  * go all UEB servers and send a get apiKeys query. In case a query is succeed, no query is sent to the rest of UEB servers.
 212                  *
 213                  *
 214                  * @return
 215                  */
 216                 private boolean queryUeb() {
 217
 218                         Boolean result = false;
 219                         int retryNumber = 1;
 220                         for (UebHealthCheckCall healthCheckCall : healthCheckCalls) {
 221                                 try {
 222
 223                                         healthLogger.debug("Before running Health Check retry query number {} towards UEB server {}", retryNumber, healthCheckCall.getServer());
 224
 225                                         Future<Boolean> future = healthCheckExecutor.submit(healthCheckCall);
 226                                         result = future.get(healthCheckReadTimeout, TimeUnit.SECONDS);
 227
 228                                         healthLogger.debug("After running Health Check retry query number {} towards UEB server {}. Result is {}", retryNumber, healthCheckCall.getServer(), result);
 229
 230                                         if (result != null && true == result.booleanValue()) {
 231                                                 break;
 232                                         }
 233
 234                                 } catch (Exception e) {
 235                                         String message = e.getMessage();
 236                                         if (message == null) {
 237                                                 message = e.getClass().getName();
 238                                         }
 239                                         healthLogger.debug("Error occured during running Health Check retry query towards UEB server {}. Result is {}", healthCheckCall.getServer(), message);
 240                                         healthLogger.trace("Error occured during running Health Check retry query towards UEB server {}. Result is {}", healthCheckCall.getServer(), message, e);
 241                                 }
 242                                 retryNumber++;
 243
 244                         }
 245
 246                         return result;
 247
 248                 }
 249
 250                 public List<UebHealthCheckCall> getHealthCheckCalls() {
 251                         return healthCheckCalls;
 252                 }
 253
 254         }
 255
 256         @PostConstruct
 257         private void init() {
 258
 259                 logger.trace("Enter init method of DistributionEngineClusterHealth");
 260
 261                 Long reconnectIntervalConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getUebHealthCheckReconnectIntervalInSeconds();
 262                 if (reconnectIntervalConfig != null) {
 263                         reconnectInterval = reconnectIntervalConfig.longValue();
 264                 }
 265                 Long healthCheckReadTimeoutConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getUebHealthCheckReadTimeout();
 266                 if (healthCheckReadTimeoutConfig != null) {
 267                         healthCheckReadTimeout = healthCheckReadTimeoutConfig.longValue();
 268                 }
 269
 270                 DistributionEngineConfiguration distributionEngineConfiguration = ConfigurationManager.getConfigurationManager().getDistributionEngineConfiguration();
 271
 272                 this.uebServers = distributionEngineConfiguration.getUebServers();
 273                 this.publicApiKey = distributionEngineConfiguration.getUebPublicKey();
 274
 275                 this.healthCheckScheduledTask = new HealthCheckScheduledTask(this.uebServers);
 276
 277                 logger.trace("Exit init method of DistributionEngineClusterHealth");
 278
 279         }
 280
 281         @PreDestroy
 282         private void destroy() {
 283
 284                 if (scheduledFuture != null) {
 285                         scheduledFuture.cancel(true);
 286                         scheduledFuture = null;
 287                 }
 288
 289                 if (healthCheckScheduler != null) {
 290                         healthCheckScheduler.shutdown();
 291                 }
 292
 293         }
 294
 295         /**
 296          * Start health check task.
 297          *
 298          * @param envNamePerStatus
 299          * @param startTask
 300          */
 301         public void startHealthCheckTask(Map<String, AtomicBoolean> envNamePerStatus, boolean startTask) {
 302                 this.envNamePerStatus = envNamePerStatus;
 303
 304                 if (startTask == true && this.scheduledFuture == null) {
 305                         this.scheduledFuture = this.healthCheckScheduler.scheduleAtFixedRate(healthCheckScheduledTask, 0, reconnectInterval, TimeUnit.SECONDS);
 306                 }
 307         }
 308
 309         public void startHealthCheckTask(Map<String, AtomicBoolean> envNamePerStatus) {
 310                 startHealthCheckTask(envNamePerStatus, true);
 311         }
 312
 313         private void logAlarm(boolean lastHealthState) {
 314                 if (lastHealthState == true) {
 315                         BeEcompErrorManager.getInstance().processEcompError(EcompErrorName.BeHealthCheckRecovery, UEB_HEALTH_CHECK_STR);
 316                         BeEcompErrorManager.getInstance().logBeHealthCheckUebClusterRecovery(UEB_HEALTH_CHECK_STR);
 317                 } else {
 318                         BeEcompErrorManager.getInstance().processEcompError(EcompErrorName.BeHealthCheckError, UEB_HEALTH_CHECK_STR);
 319                         BeEcompErrorManager.getInstance().logBeHealthCheckUebClusterError(UEB_HEALTH_CHECK_STR);
 320                 }
 321         }
 322
 323         public HealthCheckInfo getHealthCheckInfo() {
 324                 return healthCheckInfo;
 325         }
 326
 327         /**
 328          * change the health check to DISABLE
 329          */
 330         public void setHealthCheckUebIsDisabled() {
 331                 healthCheckInfo = HealthCheckInfoResult.DISABLED.getHealthCheckInfo();
 332         }
 333
 334         /**
 335          * change the health check to NOT CONFGIURED
 336          */
 337         public void setHealthCheckUebConfigurationError() {
 338                 healthCheckInfo = HealthCheckInfoResult.NOT_CONFIGURED.getHealthCheckInfo();
 339         }
 340
 341         public void setHealthCheckOkAndReportInCaseLastStateIsDown() {
 342
 343                 if (lastHealthState == true) {
 344                         return;
 345                 }
 346                 synchronized (lockOject) {
 347                         if (lastHealthState == false) {
 348                                 logger.debug("Going to update health check state to available");
 349                                 lastHealthState = true;
 350                                 healthCheckInfo = HealthCheckInfoResult.OK.getHealthCheckInfo();
 351                                 logAlarm(lastHealthState);
 352                         }
 353                 }
 354
 355         }
 356
 357 }