2 * ================================================================================
\r
4 * ================================================================================
\r
5 * Copyright (C) 2017 AT&T Intellectual Property
\r
6 * ================================================================================
\r
7 * Licensed under the Apache License, Version 2.0 (the "License");
\r
8 * you may not use this file except in compliance with the License.
\r
9 * You may obtain a copy of the License at
\r
11 * http://www.apache.org/licenses/LICENSE-2.0
\r
13 * Unless required by applicable law or agreed to in writing, software
\r
14 * distributed under the License is distributed on an "AS IS" BASIS,
\r
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
\r
16 * See the License for the specific language governing permissions and
\r
17 * limitations under the License.
\r
18 * ================================================================================
\r
20 package org.openecomp.portalapp.portal.listener;
\r
22 import java.util.List;
\r
24 import javax.annotation.PostConstruct;
\r
25 import javax.annotation.PreDestroy;
\r
27 import org.hibernate.Query;
\r
28 import org.hibernate.Session;
\r
29 import org.hibernate.SessionFactory;
\r
30 import org.openecomp.portalapp.portal.logging.aop.EPMetricsLog;
\r
31 import org.openecomp.portalapp.portal.logging.format.EPAppMessagesEnum;
\r
32 import org.openecomp.portalapp.portal.logging.logic.EPLogUtil;
\r
33 import org.openecomp.portalapp.portal.ueb.EPUebHelper;
\r
34 import org.openecomp.portalapp.portal.utils.EPCommonSystemProperties;
\r
35 import org.openecomp.portalsdk.core.logging.logic.EELFLoggerDelegate;
\r
36 import org.openecomp.portalsdk.core.util.SystemProperties;
\r
37 import org.springframework.beans.factory.annotation.Autowired;
\r
38 import org.springframework.context.annotation.EnableAspectJAutoProxy;
\r
39 import org.springframework.transaction.annotation.Transactional;
\r
42 @org.springframework.context.annotation.Configuration
\r
43 @EnableAspectJAutoProxy
\r
45 public class HealthMonitor {
\r
48 private SessionFactory sessionFactory;
\r
51 private EPUebHelper epUebHelper;
\r
53 private static boolean databaseUp;
\r
54 private static boolean uebUp;
\r
55 private static boolean frontEndUp;
\r
56 private static boolean backEndUp;
\r
57 private static boolean dbClusterStatusOk;
\r
58 private static boolean dbPermissionsOk;
\r
59 public static boolean isSuspended = false;
\r
61 Thread healthMonitorThread;
\r
63 EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(HealthMonitor.class);
\r
65 public HealthMonitor() {
\r
69 public static boolean isDatabaseUp() {
\r
73 public static boolean isClusterStatusOk() {
\r
74 return dbClusterStatusOk;
\r
77 public static boolean isDatabasePermissionsOk() {
\r
78 return dbPermissionsOk;
\r
81 public static boolean isUebUp() {
\r
85 public static boolean isFrontEndUp() {
\r
89 public static boolean isBackEndUp() {
\r
93 private void monitorEPHealth() throws InterruptedException {
\r
95 int numIntervalsDatabaseHasBeenDown = 0;
\r
96 int numIntervalsClusterNotHealthy = 0;
\r
97 int numIntervalsDatabasePermissionsIncorrect = 0;
\r
98 int numIntervalsUebHasBeenDown = 0;
\r
100 logger.debug(EELFLoggerDelegate.debugLogger, "==> Health Monitor thread started");
\r
102 long sleepInterval = (Long
\r
103 .valueOf(SystemProperties.getProperty(EPCommonSystemProperties.HEALTH_POLL_INTERVAL_SECONDS)) * 1000);
\r
104 long numIntervalsBetweenAlerts = Long
\r
105 .valueOf(SystemProperties.getProperty(EPCommonSystemProperties.HEALTHFAIL_ALERT_EVERY_X_INTERVALS));
\r
106 logger.debug(EELFLoggerDelegate.debugLogger,
\r
107 "Polling health every " + sleepInterval + " milliseconds. Alerting every "
\r
108 + (sleepInterval * numIntervalsBetweenAlerts) / 1000 + " seconds when component remains down.");
\r
112 // Get DB status. If down, signal alert once every X intervals.
\r
114 databaseUp = this.checkIfDatabaseUp();
\r
115 if (databaseUp == false) {
\r
117 if ((numIntervalsDatabaseHasBeenDown % numIntervalsBetweenAlerts) == 0) {
\r
118 // Write a Log entry that will generate an alert
\r
119 EPLogUtil.logEcompError(logger, EPAppMessagesEnum.BeHealthCheckMySqlError);
\r
120 logger.debug(EELFLoggerDelegate.debugLogger,
\r
121 "Database down, logging to error log to trigger alert.");
\r
122 numIntervalsDatabaseHasBeenDown++;
\r
124 numIntervalsDatabaseHasBeenDown = 0;
\r
128 dbClusterStatusOk = this.checkClusterStatus();
\r
129 if (dbClusterStatusOk == false) {
\r
131 if ((numIntervalsClusterNotHealthy % numIntervalsBetweenAlerts) == 0) {
\r
132 EPLogUtil.logEcompError(logger, EPAppMessagesEnum.BeHealthCheckMySqlError);
\r
133 logger.debug(EELFLoggerDelegate.debugLogger,
\r
134 "Cluster nodes appear to be down, logging to error log to trigger alert.");
\r
135 numIntervalsClusterNotHealthy++;
\r
137 numIntervalsClusterNotHealthy = 0;
\r
141 dbPermissionsOk = this.checkDatabaseAndPermissions();
\r
142 if (dbPermissionsOk == false) {
\r
144 if ((numIntervalsDatabasePermissionsIncorrect % numIntervalsBetweenAlerts) == 0) {
\r
145 EPLogUtil.logEcompError(logger, EPAppMessagesEnum.BeHealthCheckMySqlError);
\r
146 logger.debug(EELFLoggerDelegate.debugLogger,
\r
147 "Database permissions don't seem correct, logging to error log to trigger alert.");
\r
148 numIntervalsDatabasePermissionsIncorrect++;
\r
150 numIntervalsDatabasePermissionsIncorrect = 0;
\r
155 // Get UEB status. Publish a bogus message to EP inbox, if 200 OK
\r
156 // returned, status is Up.
\r
157 // If down, signal alert once every X intervals.
\r
158 // EP will ignore this bogus message.
\r
160 uebUp = this.checkIfUebUp();
\r
161 if (uebUp == false) {
\r
163 if ((numIntervalsUebHasBeenDown % numIntervalsBetweenAlerts) == 0) {
\r
164 // Write a Log entry that will generate an alert
\r
165 EPLogUtil.logEcompError(logger, EPAppMessagesEnum.BeHealthCheckUebClusterError);
\r
166 logger.debug(EELFLoggerDelegate.debugLogger, "UEB down, logging to error log to trigger alert");
\r
167 numIntervalsUebHasBeenDown++;
\r
169 numIntervalsUebHasBeenDown = 0;
\r
173 // The front end should be up because the API is called through
\r
174 // proxy front end server.
\r
177 // If the rest API called, the backend is always up
\r
181 // future nice to have...get Partner status
\r
183 // For all apps exposing a rest url, query one of the rest
\r
184 // urls(/roles?) and manage a list
\r
185 // of app name/status. We might not return back a non 200 OK in
\r
186 // health check, but we
\r
187 // could return information in the json content of a health check.
\r
191 // Get DB status. If down, signal alert once every X intervals.
\r
193 if (Thread.interrupted()) {
\r
194 logger.info(EELFLoggerDelegate.errorLogger, "==> UebMainHandler exiting");
\r
199 Thread.sleep(sleepInterval);
\r
200 } catch (InterruptedException e) {
\r
201 logger.error(EELFLoggerDelegate.errorLogger, "monitorEPHealth interrupted", e);
\r
202 Thread.currentThread().interrupt();
\r
208 public void initHealthMonitor() {
\r
210 healthMonitorThread = new Thread("EP HealthMonitor thread") {
\r
211 public void run() {
\r
214 } catch (InterruptedException e) {
\r
215 logger.debug(EELFLoggerDelegate.debugLogger, "healthMonitorThread interrupted", e);
\r
216 } catch (Exception e) {
\r
217 logger.error(EELFLoggerDelegate.errorLogger, "healthMonitorThread failed", e);
\r
221 if (healthMonitorThread != null) {
\r
222 healthMonitorThread.start();
\r
227 public void closeHealthMonitor() {
\r
228 this.healthMonitorThread.interrupt();
\r
231 private boolean checkIfDatabaseUp() {
\r
233 boolean isUp = false;
\r
235 Session localSession = null;
\r
238 localSession = sessionFactory.openSession();
\r
240 if (localSession != null) {
\r
242 String sql = "select app_name from fn_app where app_id=1";
\r
243 Query query = localSession.createSQLQuery(sql);
\r
244 @SuppressWarnings("unchecked")
\r
245 List<String> queryList = query.list();
\r
246 if (queryList != null) {
\r
249 localSession.close();
\r
251 } catch (Exception e) {
\r
252 logger.debug(EELFLoggerDelegate.debugLogger, "checkIfDatabaseUp failed", e);
\r
259 private boolean checkClusterStatus() {
\r
261 boolean isUp = false;
\r
263 Session localSession = null;
\r
266 localSession = sessionFactory.openSession();
\r
267 if (localSession != null) {
\r
268 // If all nodes are unhealthy in a cluster, this will throw an
\r
270 String sql = "select * from mysql.user";
\r
271 Query query = localSession.createSQLQuery(sql);
\r
272 @SuppressWarnings("unchecked")
\r
273 List<String> queryList = query.list();
\r
274 if (queryList != null) {
\r
278 } catch (Exception e) {
\r
279 logger.error(EELFLoggerDelegate.errorLogger, "checkClusterStatus failed", e);
\r
280 if ((e.getCause() != null) && (e.getCause().getMessage() != null)) {
\r
281 logger.error(EELFLoggerDelegate.errorLogger,
\r
282 "checkClusterStatus() exception msg = " + e.getCause().getMessage());
\r
286 if (localSession != null) {
\r
287 localSession.close();
\r
295 private boolean checkDatabaseAndPermissions() {
\r
297 boolean isUp = false;
\r
299 Session localSession = null;
\r
302 localSession = sessionFactory.openSession();
\r
303 if (localSession != null) {
\r
304 String sql = "SHOW GRANTS FOR CURRENT_USER";
\r
305 Query query = localSession.createSQLQuery(sql);
\r
306 @SuppressWarnings("unchecked")
\r
307 List<String> grantsList = query.list();
\r
308 for (String str : grantsList) {
\r
309 if ((str.toUpperCase().contains("ALL"))
\r
310 || (str.toUpperCase().contains("DELETE") && str.toUpperCase().contains("SELECT")
\r
311 && str.toUpperCase().contains("UPDATE") && str.toUpperCase().contains("INSERT"))) {
\r
316 if (isUp == false) {
\r
317 logger.error(EELFLoggerDelegate.errorLogger,
\r
318 "checkDatabaseAndPermissions() returning false. SHOW GRANTS FOR CURRENT_USER being dumped:");
\r
319 for (String str : grantsList) {
\r
320 logger.error(EELFLoggerDelegate.errorLogger, "grants output item = [" + str + "]");
\r
324 } catch (Exception e) {
\r
325 logger.error(EELFLoggerDelegate.errorLogger, "checkDatabaseAndPermissions failed", e);
\r
326 if ((e.getCause() != null) && (e.getCause().getMessage() != null)) {
\r
327 logger.error(EELFLoggerDelegate.errorLogger,
\r
328 "checkDatabaseAndPermissions() exception msg = " + e.getCause().getMessage());
\r
332 if (localSession != null) {
\r
333 localSession.close();
\r
341 private boolean checkIfUebUp() {
\r
342 boolean uebUp = false;
\r
344 boolean isAvailable = epUebHelper.checkAvailability();
\r
345 boolean messageCanBeSent = epUebHelper.MessageCanBeSentToTopic();
\r
346 uebUp = (isAvailable && messageCanBeSent);
\r
347 } catch (Exception e) {
\r
348 logger.error(EELFLoggerDelegate.errorLogger, "checkIfUebUp failed", e);
\r