gerrit.onap Code Review - sdc.git/blob

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.be.components.distribution.engine;
  22
  23 import java.util.ArrayList;
  24 import java.util.Collection;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.concurrent.ExecutorService;
  28 import java.util.concurrent.Executors;
  29 import java.util.concurrent.Future;
  30 import java.util.concurrent.ScheduledExecutorService;
  31 import java.util.concurrent.ScheduledFuture;
  32 import java.util.concurrent.ThreadFactory;
  33 import java.util.concurrent.TimeUnit;
  34 import java.util.concurrent.atomic.AtomicBoolean;
  35
  36 import javax.annotation.PostConstruct;
  37 import javax.annotation.PreDestroy;
  38
  39 import org.openecomp.sdc.be.config.BeEcompErrorManager;
  40 import org.openecomp.sdc.be.config.ConfigurationManager;
  41 import org.openecomp.sdc.be.config.DistributionEngineConfiguration;
  42 import org.openecomp.sdc.common.api.HealthCheckInfo;
  43 import org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckComponent;
  44 import org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus;
  45 import org.openecomp.sdc.common.config.EcompErrorName;
  46 import org.slf4j.Logger;
  47 import org.slf4j.LoggerFactory;
  48 import org.springframework.stereotype.Component;
  49
  50 @Component("distribution-engine-cluster-health")
  51 public class DistributionEngineClusterHealth {
  52
  53         protected static String UEB_HEALTH_LOG_CONTEXT = "ueb.healthcheck";
  54
  55         private static Logger healthLogger = LoggerFactory.getLogger(UEB_HEALTH_LOG_CONTEXT);
  56
  57         private static final String UEB_HEALTH_CHECK_STR = "uebHealthCheck";
  58
  59         boolean lastHealthState = false;
  60
  61         Object lockOject = new Object();
  62
  63         private long reconnectInterval = 5;
  64
  65         private long healthCheckReadTimeout = 20;
  66
  67         private static Logger logger = LoggerFactory.getLogger(DistributionEngineClusterHealth.class.getName());
  68
  69         private List<String> uebServers = null;
  70
  71         private String publicApiKey = null;
  72
  73         public enum HealthCheckInfoResult {
  74
  75                 OK(new HealthCheckInfo(HealthCheckComponent.DE, HealthCheckStatus.UP, null, ClusterStatusDescription.OK.getDescription())), UNAVAILABLE(
  76                                 new HealthCheckInfo(HealthCheckComponent.DE, HealthCheckStatus.DOWN, null, ClusterStatusDescription.UNAVAILABLE.getDescription())), NOT_CONFIGURED(
  77                                                 new HealthCheckInfo(HealthCheckComponent.DE, HealthCheckStatus.DOWN, null, ClusterStatusDescription.NOT_CONFIGURED.getDescription())), DISABLED(
  78                                                                 new HealthCheckInfo(HealthCheckComponent.DE, HealthCheckStatus.DOWN, null, ClusterStatusDescription.DISABLED.getDescription()));
  79
  80                 private HealthCheckInfo healthCheckInfo;
  81
  82                 HealthCheckInfoResult(HealthCheckInfo healthCheckInfo) {
  83                         this.healthCheckInfo = healthCheckInfo;
  84                 }
  85
  86                 public HealthCheckInfo getHealthCheckInfo() {
  87                         return healthCheckInfo;
  88                 }
  89
  90         }
  91
  92         private HealthCheckInfo healthCheckInfo = HealthCheckInfoResult.UNAVAILABLE.getHealthCheckInfo();
  93
  94         private Map<String, AtomicBoolean> envNamePerStatus = null;
  95
  96         private ScheduledFuture<?> scheduledFuture = null;
  97
  98         ScheduledExecutorService healthCheckScheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
  99                 @Override
 100                 public Thread newThread(Runnable r) {
 101                         return new Thread(r, "UEB-Health-Check-Task");
 102                 }
 103         });
 104
 105         HealthCheckScheduledTask healthCheckScheduledTask = null;
 106
 107         public enum ClusterStatusDescription {
 108
 109                 OK("OK"), UNAVAILABLE("U-EB cluster is not available"), NOT_CONFIGURED("U-EB cluster is not configured"), DISABLED("DE is disabled in configuration");
 110
 111                 private String desc;
 112
 113                 ClusterStatusDescription(String desc) {
 114                         this.desc = desc;
 115                 }
 116
 117                 public String getDescription() {
 118                         return desc;
 119                 }
 120
 121         }
 122
 123         /**
 124          * Health Check Task Scheduler.
 125          *
 126          * It schedules a task which send a apiKey get query towards the UEB servers. In case a query to the first UEB server is failed, then a second query is sent to the next UEB server.
 127          *
 128          *
 129          * @author esofer
 130          *
 131          */
 132         public class HealthCheckScheduledTask implements Runnable {
 133
 134                 List<UebHealthCheckCall> healthCheckCalls = new ArrayList<>();
 135
 136                 public HealthCheckScheduledTask(List<String> uebServers) {
 137
 138                         logger.debug("Create health check calls for servers {}", uebServers);
 139                         if (uebServers != null) {
 140                                 for (String server : uebServers) {
 141                                         healthCheckCalls.add(new UebHealthCheckCall(server, publicApiKey));
 142                                 }
 143                         }
 144                 }
 145
 146                 @Override
 147                 public void run() {
 148
 149                         healthLogger.trace("Executing UEB Health Check Task - Start");
 150
 151                         boolean healthStatus = verifyAtLeastOneEnvIsUp();
 152
 153                         if (true == healthStatus) {
 154                                 boolean queryUebStatus = queryUeb();
 155                                 if (queryUebStatus == lastHealthState) {
 156                                         return;
 157                                 }
 158
 159                                 synchronized (lockOject) {
 160                                         if (queryUebStatus != lastHealthState) {
 161                                                 logger.trace("UEB Health State Changed to {}. Issuing alarm / recovery alarm...", healthStatus);
 162                                                 lastHealthState = queryUebStatus;
 163                                                 logAlarm(lastHealthState);
 164                                                 if (true == queryUebStatus) {
 165                                                         healthCheckInfo = HealthCheckInfoResult.OK.getHealthCheckInfo();
 166                                                 } else {
 167                                                         healthCheckInfo = HealthCheckInfoResult.UNAVAILABLE.getHealthCheckInfo();
 168                                                 }
 169                                         }
 170                                 }
 171                         } else {
 172                                 healthLogger.trace("Not all UEB Environments are up");
 173                         }
 174
 175                 }
 176
 177                 /**
 178                  * verify that at least one environment is up.
 179                  *
 180                  */
 181                 private boolean verifyAtLeastOneEnvIsUp() {
 182
 183                         boolean healthStatus = false;
 184
 185                         if (envNamePerStatus != null) {
 186                                 Collection<AtomicBoolean> values = envNamePerStatus.values();
 187                                 if (values != null) {
 188                                         for (AtomicBoolean status : values) {
 189                                                 if (true == status.get()) {
 190                                                         healthStatus = true;
 191                                                         break;
 192                                                 }
 193                                         }
 194                                 }
 195                         }
 196
 197                         return healthStatus;
 198                 }
 199
 200                 /**
 201                  * executor for the query itself
 202                  */
 203                 ExecutorService healthCheckExecutor = Executors.newSingleThreadExecutor(new ThreadFactory() {
 204                         @Override
 205                         public Thread newThread(Runnable r) {
 206                                 return new Thread(r, "UEB-Health-Check-Thread");
 207                         }
 208                 });
 209
 210                 /**
 211                  * go all UEB servers and send a get apiKeys query. In case a query is succeed, no query is sent to the rest of UEB servers.
 212                  *
 213                  *
 214                  * @return
 215                  */
 216                 private boolean queryUeb() {
 217
 218                         Boolean result = false;
 219                         int retryNumber = 1;
 220                         for (UebHealthCheckCall healthCheckCall : healthCheckCalls) {
 221                                 try {
 222
 223                                         healthLogger.debug("Before running Health Check retry query number {} towards UEB server {}", retryNumber, healthCheckCall.getServer());
 224
 225                                         Future<Boolean> future = healthCheckExecutor.submit(healthCheckCall);
 226                                         result = future.get(healthCheckReadTimeout, TimeUnit.SECONDS);
 227
 228                                         healthLogger.debug("After running Health Check retry query number {} towards UEB server {}. Result is {}", retryNumber, healthCheckCall.getServer(), result);
 229
 230                                         if (result != null && true == result.booleanValue()) {
 231                                                 break;
 232                                         }
 233
 234                                 } catch (Exception e) {
 235                                         String message = e.getMessage();
 236                                         if (message == null) {
 237                                                 message = e.getClass().getName();
 238                                         }
 239                                         healthLogger.debug("Error occured during running Health Check retry query towards UEB server {}. Result is {}", healthCheckCall.getServer(), message);
 240                                 }
 241                                 retryNumber++;
 242
 243                         }
 244
 245                         return result;
 246
 247                 }
 248
 249                 public List<UebHealthCheckCall> getHealthCheckCalls() {
 250                         return healthCheckCalls;
 251                 }
 252
 253         }
 254
 255         @PostConstruct
 256         private void init() {
 257
 258                 logger.trace("Enter init method of DistributionEngineClusterHealth");
 259
 260                 Long reconnectIntervalConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getUebHealthCheckReconnectIntervalInSeconds();
 261                 if (reconnectIntervalConfig != null) {
 262                         reconnectInterval = reconnectIntervalConfig.longValue();
 263                 }
 264                 Long healthCheckReadTimeoutConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getUebHealthCheckReadTimeout();
 265                 if (healthCheckReadTimeoutConfig != null) {
 266                         healthCheckReadTimeout = healthCheckReadTimeoutConfig.longValue();
 267                 }
 268
 269                 DistributionEngineConfiguration distributionEngineConfiguration = ConfigurationManager.getConfigurationManager().getDistributionEngineConfiguration();
 270
 271                 this.uebServers = distributionEngineConfiguration.getUebServers();
 272                 this.publicApiKey = distributionEngineConfiguration.getUebPublicKey();
 273
 274                 this.healthCheckScheduledTask = new HealthCheckScheduledTask(this.uebServers);
 275
 276                 logger.trace("Exit init method of DistributionEngineClusterHealth");
 277
 278         }
 279
 280         @PreDestroy
 281         private void destroy() {
 282
 283                 if (scheduledFuture != null) {
 284                         scheduledFuture.cancel(true);
 285                         scheduledFuture = null;
 286                 }
 287
 288                 if (healthCheckScheduler != null) {
 289                         healthCheckScheduler.shutdown();
 290                 }
 291
 292         }
 293
 294         /**
 295          * Start health check task.
 296          *
 297          * @param envNamePerStatus
 298          * @param startTask
 299          */
 300         public void startHealthCheckTask(Map<String, AtomicBoolean> envNamePerStatus, boolean startTask) {
 301                 this.envNamePerStatus = envNamePerStatus;
 302
 303                 if (startTask == true && this.scheduledFuture == null) {
 304                         this.scheduledFuture = this.healthCheckScheduler.scheduleAtFixedRate(healthCheckScheduledTask, 0, reconnectInterval, TimeUnit.SECONDS);
 305                 }
 306         }
 307
 308         public void startHealthCheckTask(Map<String, AtomicBoolean> envNamePerStatus) {
 309                 startHealthCheckTask(envNamePerStatus, true);
 310         }
 311
 312         private void logAlarm(boolean lastHealthState) {
 313                 if (lastHealthState == true) {
 314                         BeEcompErrorManager.getInstance().processEcompError(EcompErrorName.BeHealthCheckRecovery, UEB_HEALTH_CHECK_STR);
 315                         BeEcompErrorManager.getInstance().logBeHealthCheckUebClusterRecovery(UEB_HEALTH_CHECK_STR);
 316                 } else {
 317                         BeEcompErrorManager.getInstance().processEcompError(EcompErrorName.BeHealthCheckError, UEB_HEALTH_CHECK_STR);
 318                         BeEcompErrorManager.getInstance().logBeHealthCheckUebClusterError(UEB_HEALTH_CHECK_STR);
 319                 }
 320         }
 321
 322         public HealthCheckInfo getHealthCheckInfo() {
 323                 return healthCheckInfo;
 324         }
 325
 326         /**
 327          * change the health check to DISABLE
 328          */
 329         public void setHealthCheckUebIsDisabled() {
 330                 healthCheckInfo = HealthCheckInfoResult.DISABLED.getHealthCheckInfo();
 331         }
 332
 333         /**
 334          * change the health check to NOT CONFGIURED
 335          */
 336         public void setHealthCheckUebConfigurationError() {
 337                 healthCheckInfo = HealthCheckInfoResult.NOT_CONFIGURED.getHealthCheckInfo();
 338         }
 339
 340         public void setHealthCheckOkAndReportInCaseLastStateIsDown() {
 341
 342                 if (lastHealthState == true) {
 343                         return;
 344                 }
 345                 synchronized (lockOject) {
 346                         if (lastHealthState == false) {
 347                                 logger.debug("Going to update health check state to available");
 348                                 lastHealthState = true;
 349                                 healthCheckInfo = HealthCheckInfoResult.OK.getHealthCheckInfo();
 350                                 logAlarm(lastHealthState);
 351                         }
 352                 }
 353
 354         }
 355
 356 }