48533f1518db5ab9d10089c9ed25c76273ae80c8
[sdc.git] /
1 /*-
2  * ============LICENSE_START=======================================================
3  * SDC
4  * ================================================================================
5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
6  * ================================================================================
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  * 
11  *      http://www.apache.org/licenses/LICENSE-2.0
12  * 
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  * ============LICENSE_END=========================================================
19  */
20
21 package org.openecomp.sdc.be.components.distribution.engine;
22
23 import java.util.ArrayList;
24 import java.util.Collection;
25 import java.util.List;
26 import java.util.Map;
27 import java.util.concurrent.ExecutorService;
28 import java.util.concurrent.Executors;
29 import java.util.concurrent.Future;
30 import java.util.concurrent.ScheduledExecutorService;
31 import java.util.concurrent.ScheduledFuture;
32 import java.util.concurrent.ThreadFactory;
33 import java.util.concurrent.TimeUnit;
34 import java.util.concurrent.atomic.AtomicBoolean;
35
36 import javax.annotation.PostConstruct;
37 import javax.annotation.PreDestroy;
38
39 import org.openecomp.sdc.be.config.BeEcompErrorManager;
40 import org.openecomp.sdc.be.config.ConfigurationManager;
41 import org.openecomp.sdc.be.config.DistributionEngineConfiguration;
42 import org.openecomp.sdc.common.api.Constants;
43 import org.openecomp.sdc.common.api.HealthCheckInfo;
44 import org.openecomp.sdc.common.api.HealthCheckInfo.HealthCheckStatus;
45 import org.slf4j.Logger;
46 import org.slf4j.LoggerFactory;
47 import org.springframework.stereotype.Component;
48
49 @Component("distribution-engine-cluster-health")
50 public class DistributionEngineClusterHealth {
51
52     protected static String UEB_HEALTH_LOG_CONTEXT = "ueb.healthcheck";
53
54     private static final Logger healthLogger = LoggerFactory.getLogger(UEB_HEALTH_LOG_CONTEXT);
55
56     private static final String UEB_HEALTH_CHECK_STR = "uebHealthCheck";
57
58     boolean lastHealthState = false;
59
60     Object lockOject = new Object();
61
62     private long reconnectInterval = 5;
63
64     private long healthCheckReadTimeout = 20;
65
66     private static final Logger logger = LoggerFactory.getLogger(DistributionEngineClusterHealth.class);
67
68     private List<String> uebServers = null;
69
70     private String publicApiKey = null;
71
72     public enum HealthCheckInfoResult {
73
74         OK(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.UP, null, ClusterStatusDescription.OK.getDescription())),
75         UNAVAILABLE(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.DOWN, null, ClusterStatusDescription.UNAVAILABLE.getDescription())),
76         NOT_CONFIGURED(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.DOWN, null, ClusterStatusDescription.NOT_CONFIGURED.getDescription())),
77         DISABLED(new HealthCheckInfo(Constants.HC_COMPONENT_DISTRIBUTION_ENGINE, HealthCheckStatus.DOWN, null, ClusterStatusDescription.DISABLED.getDescription()));
78
79         private HealthCheckInfo healthCheckInfo;
80
81         HealthCheckInfoResult(HealthCheckInfo healthCheckInfo) {
82             this.healthCheckInfo = healthCheckInfo;
83         }
84
85         public HealthCheckInfo getHealthCheckInfo() {
86             return healthCheckInfo;
87         }
88
89     }
90
91     private HealthCheckInfo healthCheckInfo = HealthCheckInfoResult.UNAVAILABLE.getHealthCheckInfo();
92
93     private Map<String, AtomicBoolean> envNamePerStatus = null;
94
95     private ScheduledFuture<?> scheduledFuture = null;
96
97     ScheduledExecutorService healthCheckScheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
98         @Override
99         public Thread newThread(Runnable r) {
100             return new Thread(r, "UEB-Health-Check-Task");
101         }
102     });
103
104     HealthCheckScheduledTask healthCheckScheduledTask = null;
105
106     public enum ClusterStatusDescription {
107
108         OK("OK"), UNAVAILABLE("U-EB cluster is not available"), NOT_CONFIGURED("U-EB cluster is not configured"), DISABLED("DE is disabled in configuration");
109
110         private String desc;
111
112         ClusterStatusDescription(String desc) {
113             this.desc = desc;
114         }
115
116         public String getDescription() {
117             return desc;
118         }
119
120     }
121
122     /**
123      * Health Check Task Scheduler.
124      *
125      * It schedules a task which send a apiKey get query towards the UEB servers. In case a query to the first UEB server is failed, then a second query is sent to the next UEB server.
126      *
127      *
128      * @author esofer
129      *
130      */
131     public class HealthCheckScheduledTask implements Runnable {
132
133         List<UebHealthCheckCall> healthCheckCalls = new ArrayList<>();
134
135         public HealthCheckScheduledTask(List<String> uebServers) {
136
137             logger.debug("Create health check calls for servers {}", uebServers);
138             if (uebServers != null) {
139                 for (String server : uebServers) {
140                     healthCheckCalls.add(new UebHealthCheckCall(server, publicApiKey));
141                 }
142             }
143         }
144
145         @Override
146         public void run() {
147
148             healthLogger.trace("Executing UEB Health Check Task - Start");
149
150             boolean healthStatus = verifyAtLeastOneEnvIsUp();
151
152             if (true == healthStatus) {
153                 boolean queryUebStatus = queryUeb();
154                 if (queryUebStatus == lastHealthState) {
155                     return;
156                 }
157
158                 synchronized (lockOject) {
159                     if (queryUebStatus != lastHealthState) {
160                         logger.trace("UEB Health State Changed to {}. Issuing alarm / recovery alarm...", healthStatus);
161                         lastHealthState = queryUebStatus;
162                         logAlarm(lastHealthState);
163                         if (true == queryUebStatus) {
164                             healthCheckInfo = HealthCheckInfoResult.OK.getHealthCheckInfo();
165                         } else {
166                             healthCheckInfo = HealthCheckInfoResult.UNAVAILABLE.getHealthCheckInfo();
167                         }
168                     }
169                 }
170             } else {
171                 healthLogger.trace("Not all UEB Environments are up");
172             }
173
174         }
175
176         /**
177          * verify that at least one environment is up.
178          *
179          */
180         private boolean verifyAtLeastOneEnvIsUp() {
181
182             boolean healthStatus = false;
183
184             if (envNamePerStatus != null) {
185                 Collection<AtomicBoolean> values = envNamePerStatus.values();
186                 if (values != null) {
187                     for (AtomicBoolean status : values) {
188                         if (true == status.get()) {
189                             healthStatus = true;
190                             break;
191                         }
192                     }
193                 }
194             }
195
196             return healthStatus;
197         }
198
199         /**
200          * executor for the query itself
201          */
202         ExecutorService healthCheckExecutor = Executors.newSingleThreadExecutor(new ThreadFactory() {
203             @Override
204             public Thread newThread(Runnable r) {
205                 return new Thread(r, "UEB-Health-Check-Thread");
206             }
207         });
208
209         /**
210          * go all UEB servers and send a get apiKeys query. In case a query is succeed, no query is sent to the rest of UEB servers.
211          *
212          *
213          * @return
214          */
215         private boolean queryUeb() {
216
217             Boolean result = false;
218             int retryNumber = 1;
219             for (UebHealthCheckCall healthCheckCall : healthCheckCalls) {
220                 try {
221
222                     healthLogger.debug("Before running Health Check retry query number {} towards UEB server {}", retryNumber, healthCheckCall.getServer());
223
224                     Future<Boolean> future = healthCheckExecutor.submit(healthCheckCall);
225                     result = future.get(healthCheckReadTimeout, TimeUnit.SECONDS);
226
227                     healthLogger.debug("After running Health Check retry query number {} towards UEB server {}. Result is {}", retryNumber, healthCheckCall.getServer(), result);
228
229                     if (result != null && true == result.booleanValue()) {
230                         break;
231                     }
232
233                 } catch (Exception e) {
234                     String message = e.getMessage();
235                     if (message == null) {
236                         message = e.getClass().getName();
237                     }
238                     healthLogger.debug("Error occured during running Health Check retry query towards UEB server {}. Result is {}", healthCheckCall.getServer(), message);
239                     healthLogger.trace("Error occured during running Health Check retry query towards UEB server {}. Result is {}", healthCheckCall.getServer(), message, e);
240                 }
241                 retryNumber++;
242
243             }
244
245             return result;
246
247         }
248
249         public List<UebHealthCheckCall> getHealthCheckCalls() {
250             return healthCheckCalls;
251         }
252
253     }
254
255     @PostConstruct
256     protected void init() {
257
258         logger.trace("Enter init method of DistributionEngineClusterHealth");
259
260         Long reconnectIntervalConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getUebHealthCheckReconnectIntervalInSeconds();
261         if (reconnectIntervalConfig != null) {
262             reconnectInterval = reconnectIntervalConfig.longValue();
263         }
264         Long healthCheckReadTimeoutConfig = ConfigurationManager.getConfigurationManager().getConfiguration().getUebHealthCheckReadTimeout();
265         if (healthCheckReadTimeoutConfig != null) {
266             healthCheckReadTimeout = healthCheckReadTimeoutConfig.longValue();
267         }
268
269         DistributionEngineConfiguration distributionEngineConfiguration = ConfigurationManager.getConfigurationManager().getDistributionEngineConfiguration();
270
271         this.uebServers = distributionEngineConfiguration.getUebServers();
272         this.publicApiKey = distributionEngineConfiguration.getUebPublicKey();
273
274         this.healthCheckScheduledTask = new HealthCheckScheduledTask(this.uebServers);
275
276         logger.trace("Exit init method of DistributionEngineClusterHealth");
277
278     }
279
280     @PreDestroy
281     protected void destroy() {
282
283         if (scheduledFuture != null) {
284             scheduledFuture.cancel(true);
285             scheduledFuture = null;
286         }
287
288         if (healthCheckScheduler != null) {
289             healthCheckScheduler.shutdown();
290         }
291
292     }
293
294     /**
295      * Start health check task.
296      *
297      * @param envNamePerStatus
298      * @param startTask
299      */
300     public void startHealthCheckTask(Map<String, AtomicBoolean> envNamePerStatus, boolean startTask) {
301         this.envNamePerStatus = envNamePerStatus;
302
303         if (startTask == true && this.scheduledFuture == null) {
304             this.scheduledFuture = this.healthCheckScheduler.scheduleAtFixedRate(healthCheckScheduledTask, 0, reconnectInterval, TimeUnit.SECONDS);
305         }
306     }
307
308     public void startHealthCheckTask(Map<String, AtomicBoolean> envNamePerStatus) {
309         startHealthCheckTask(envNamePerStatus, true);
310     }
311
312     private void logAlarm(boolean lastHealthState) {
313         if (lastHealthState == true) {
314             BeEcompErrorManager.getInstance().logBeHealthCheckUebClusterRecovery(UEB_HEALTH_CHECK_STR);
315         } else {
316             BeEcompErrorManager.getInstance().logBeHealthCheckUebClusterError(UEB_HEALTH_CHECK_STR);
317         }
318     }
319
320     public HealthCheckInfo getHealthCheckInfo() {
321         return healthCheckInfo;
322     }
323
324     /**
325      * change the health check to DISABLE
326      */
327     public void setHealthCheckUebIsDisabled() {
328         healthCheckInfo = HealthCheckInfoResult.DISABLED.getHealthCheckInfo();
329     }
330
331     /**
332      * change the health check to NOT CONFGIURED
333      */
334     public void setHealthCheckUebConfigurationError() {
335         healthCheckInfo = HealthCheckInfoResult.NOT_CONFIGURED.getHealthCheckInfo();
336     }
337
338     public void setHealthCheckOkAndReportInCaseLastStateIsDown() {
339
340         if (lastHealthState == true) {
341             return;
342         }
343         synchronized (lockOject) {
344             if (lastHealthState == false) {
345                 logger.debug("Going to update health check state to available");
346                 lastHealthState = true;
347                 healthCheckInfo = HealthCheckInfoResult.OK.getHealthCheckInfo();
348                 logAlarm(lastHealthState);
349             }
350         }
351
352     }
353
354 }