dblib/provider/src/main/java/org/onap/ccsdk/sli/core/dblib/DBResourceManager.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * onap
   4  * ================================================================================
   5  * Copyright (C) 2016 - 2017 ONAP
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.ccsdk.sli.core.dblib;
  22
  23 import java.io.PrintWriter;
  24 import java.sql.Connection;
  25 import java.sql.SQLDataException;
  26 import java.sql.SQLException;
  27 import java.sql.SQLFeatureNotSupportedException;
  28 import java.sql.SQLIntegrityConstraintViolationException;
  29 import java.sql.SQLSyntaxErrorException;
  30 import java.util.ArrayList;
  31 import java.util.Collections;
  32 import java.util.Comparator;
  33 import java.util.HashSet;
  34 import java.util.Iterator;
  35 import java.util.LinkedList;
  36 import java.util.Observable;
  37 import java.util.PriorityQueue;
  38 import java.util.Properties;
  39 import java.util.Queue;
  40 import java.util.Set;
  41 import java.util.Timer;
  42 import java.util.TimerTask;
  43 import java.util.concurrent.ConcurrentLinkedQueue;
  44 import java.util.concurrent.atomic.AtomicBoolean;
  45
  46 import javax.sql.DataSource;
  47 import javax.sql.rowset.CachedRowSet;
  48
  49 import org.apache.tomcat.jdbc.pool.PoolExhaustedException;
  50 import org.onap.ccsdk.sli.core.dblib.config.DbConfigPool;
  51 import org.onap.ccsdk.sli.core.dblib.factory.AbstractDBResourceManagerFactory;
  52 import org.onap.ccsdk.sli.core.dblib.factory.AbstractResourceManagerFactory;
  53 import org.onap.ccsdk.sli.core.dblib.factory.DBConfigFactory;
  54 import org.onap.ccsdk.sli.core.dblib.pm.PollingWorker;
  55 import org.onap.ccsdk.sli.core.dblib.pm.SQLExecutionMonitor;
  56
  57 import com.mysql.jdbc.exceptions.jdbc4.MySQLNonTransientConnectionException;
  58 import org.slf4j.Logger;
  59 import org.slf4j.LoggerFactory;
  60
  61
  62 /**
  63  * @version $Revision: 1.15 $
  64  * Change Log
  65  * Author         Date     Comments
  66  * ============== ======== ====================================================
  67  * Rich Tabedzki
  68  */
  69 public class DBResourceManager implements DataSource, DataAccessor, DBResourceObserver, DbLibService {
  70         private static Logger LOGGER = LoggerFactory.getLogger(DBResourceManager.class);
  71
  72         transient boolean terminating = false;
  73         transient protected long retryInterval = 10000L;
  74         transient boolean recoveryMode = true;
  75
  76         protected final AtomicBoolean dsSelector = new  AtomicBoolean();
  77
  78 //      Queue<CachedDataSource> dsQueue = new ConcurrentLinkedQueue<CachedDataSource>();
  79         Queue<CachedDataSource> dsQueue = new PriorityQueue<CachedDataSource>(4, new Comparator<CachedDataSource>(){
  80
  81                 @Override
  82                 public int compare(CachedDataSource left, CachedDataSource right) {
  83                         try {
  84                                 if(!left.isSlave())
  85                                         return -1;
  86                                 if(!right.isSlave())
  87                                         return 1;
  88
  89                         } catch (Throwable e) {
  90                                 LOGGER.warn("", e);
  91                         }
  92                         return 0;
  93                 }
  94
  95         });
  96         protected final Set<CachedDataSource> broken = Collections.synchronizedSet(new HashSet<CachedDataSource>());
  97         protected final Object monitor = new Object();
  98         protected final Properties configProps;
  99         protected final Thread worker;
 100
 101         protected final long terminationTimeOut;
 102         protected final boolean monitorDbResponse;
 103         protected final long monitoringInterval;
 104         protected final long monitoringInitialDelay;
 105         protected final long expectedCompletionTime;
 106         protected final long unprocessedFailoverThreshold;
 107
 108         public DBResourceManager(Properties props){
 109                 this.configProps = props;
 110
 111                 // get retry interval value
 112                 retryInterval = getLongFromProperties(props, "org.onap.dblib.connection.retry", 10000L);
 113
 114                 // get recovery mode flag
 115                 recoveryMode = getBooleanFromProperties(props, "org.onap.dblib.connection.recovery", true);
 116                 if(!recoveryMode)
 117                 {
 118                         recoveryMode = false;
 119                         LOGGER.info("Recovery Mode disabled");
 120                 }
 121                 // get time out value for thread cleanup
 122                 terminationTimeOut = getLongFromProperties(props, "org.onap.dblib.termination.timeout", 300000L);
 123                 // get properties for monitoring
 124                 monitorDbResponse = getBooleanFromProperties(props, "org.onap.dblib.connection.monitor", false);
 125                 monitoringInterval = getLongFromProperties(props, "org.onap.dblib.connection.monitor.interval", 1000L);
 126                 monitoringInitialDelay = getLongFromProperties(props, "org.onap.dblib.connection.monitor.startdelay", 5000L);
 127                 expectedCompletionTime = getLongFromProperties(props, "org.onap.dblib.connection.monitor.expectedcompletiontime", 5000L);
 128                 unprocessedFailoverThreshold = getLongFromProperties(props, "org.onap.dblib.connection.monitor.unprocessedfailoverthreshold", 3L);
 129
 130                 // initialize performance monitor
 131                 PollingWorker.createInistance(props);
 132
 133                 // initialize recovery thread
 134                 worker = new RecoveryMgr();
 135                 worker.setName("DBResourcemanagerWatchThread");
 136                 worker.setDaemon(true);
 137                 worker.start();
 138         }
 139
 140         private void config(Properties ctx) throws Exception {
 141
 142                 DbConfigPool dbConfig = DBConfigFactory.createConfig(this.configProps);
 143
 144                 try {
 145                         AbstractResourceManagerFactory factory =  AbstractDBResourceManagerFactory.getFactory(dbConfig.getType());
 146                         if(LOGGER.isInfoEnabled()){
 147                                 LOGGER.info("Default DB config is : " + dbConfig.getType());
 148                                 LOGGER.info("Using factory : " + factory.getClass().getName());
 149                         }
 150                         CachedDataSource[] cachedDS = factory.initDBResourceManager(dbConfig, this);
 151                         if(cachedDS == null || cachedDS.length == 0) {
 152                                 LOGGER.error("Initialization of CachedDataSources failed. No instance was created.");
 153                                 throw new Exception("Failed to initialize DB Library. No data source was created.");
 154                         }
 155
 156                         for(int i=0; i<cachedDS.length; i++){
 157                                 if(cachedDS[i] != null && cachedDS[i].isInitialized()){
 158                                         setDataSource(cachedDS[i]);
 159                                         cachedDS[i].setInterval(monitoringInterval);
 160                                         cachedDS[i].setInitialDelay(monitoringInitialDelay);
 161                                         cachedDS[i].setExpectedCompletionTime(expectedCompletionTime);
 162                                         cachedDS[i].setUnprocessedFailoverThreshold(unprocessedFailoverThreshold);
 163                                         cachedDS[i].addObserver(this);
 164                                 }
 165                         }
 166
 167                 } catch(Exception exc){
 168                         throw exc;
 169                 }
 170         }
 171
 172         private long getLongFromProperties(Properties props, String property, long defaultValue)
 173         {
 174                 String value = null;
 175                 long tmpLongValue = defaultValue;
 176                 try {
 177                         value = (String)props.getProperty(property);
 178                         if(value != null)
 179                                 tmpLongValue = Long.parseLong(value);
 180
 181                 } catch(NumberFormatException exc) {
 182                         if(LOGGER.isWarnEnabled()){
 183                                 LOGGER.warn("'"+property+"'=" + value+" is invalid. It should be a numeric value");
 184                         }
 185                 } catch(Exception exc) {
 186                 }
 187                 return tmpLongValue;
 188
 189         }
 190
 191         private boolean getBooleanFromProperties(Properties props, String property, boolean defaultValue)
 192         {
 193                 boolean tmpValue = defaultValue;
 194                 String value = null;
 195
 196                 try {
 197                         value = (String)props.getProperty(property);
 198                         if(value != null)
 199                                 tmpValue = Boolean.parseBoolean(value);
 200
 201                 } catch(NumberFormatException exc) {
 202                         if(LOGGER.isWarnEnabled()){
 203                                 LOGGER.warn("'"+property+"'=" + value+" is invalid. It should be a boolean value");
 204                         }
 205                 } catch(Exception exc) {
 206                 }
 207                 return tmpValue;
 208
 209         }
 210
 211
 212         public void update(Observable observable, Object data) {
 213                 // if observable is active and there is a standby available, switch
 214                 if(observable instanceof SQLExecutionMonitor)
 215                 {
 216                         SQLExecutionMonitor monitor = (SQLExecutionMonitor)observable;
 217                         if(monitor.getParent() instanceof CachedDataSource)
 218                         {
 219                                 CachedDataSource dataSource = (CachedDataSource)monitor.getParent();
 220                                 if(dataSource == dsQueue.peek())
 221                                 {
 222                                         if(recoveryMode && dsQueue.size() > 1){
 223                                                 handleGetConnectionException(dataSource, new Exception(data.toString()));
 224                                         }
 225                                 }
 226                         }
 227                 }
 228         }
 229
 230         public void testForceRecovery()
 231         {
 232                 CachedDataSource active = (CachedDataSource) this.dsQueue.peek();
 233                 handleGetConnectionException(active, new Exception("test"));
 234         }
 235
 236         class RecoveryMgr extends Thread {
 237
 238                 public void run() {
 239                         while(!terminating)
 240                         {
 241                                 try {
 242                                         Thread.sleep(retryInterval);
 243                                 } catch (InterruptedException e1) {     }
 244                                 CachedDataSource brokenSource = null;
 245                                 try {
 246                                         if (!broken.isEmpty()) {
 247                                                 CachedDataSource[] sourceArray = broken.toArray(new CachedDataSource[0]);
 248                                                 for (int i = 0; i < sourceArray.length; i++)
 249                                                 {
 250                                                         brokenSource = sourceArray[i];
 251                                                         if (brokenSource instanceof TerminatingCachedDataSource)
 252                                                                 break;
 253                                                         if (resetConnectionPool(brokenSource)) {
 254                                                                 broken.remove(brokenSource);
 255                                                                 brokenSource.blockImmediateOffLine();
 256                                                                 dsQueue.add(brokenSource);
 257                                                                 LOGGER.info("DataSource <"
 258                                                                                 + brokenSource.getDbConnectionName()
 259                                                                                 + "> recovered.");
 260                                                         }
 261                                                         brokenSource = null;
 262                                                 }
 263                                         }
 264                                 } catch (Exception exc) {
 265                                         LOGGER.warn(exc.getMessage());
 266                                         if(brokenSource != null){
 267                                                 try {
 268                                                         if(!broken.contains(brokenSource))
 269                                                                 broken.add(brokenSource);
 270                                                         brokenSource = null;
 271                                                 } catch (Exception e1) {        }
 272                                         }
 273                                 }
 274                         }
 275                         LOGGER.info("DBResourceManager.RecoveryMgr <"+this.toString() +"> terminated." );
 276                 }
 277
 278                 private boolean resetConnectionPool(CachedDataSource dataSource){
 279                         try {
 280                                 return dataSource.testConnection();
 281                         } catch (Exception exc) {
 282                                 LOGGER.info("DataSource <" + dataSource.getDbConnectionName() + "> resetCache failed with error: "+ exc.getMessage());
 283                                 return false;
 284                         }
 285                 }
 286         }
 287
 288         /* (non-Javadoc)
 289          * @see org.onap.ccsdk.sli.resource.dblib.DbLibService#getData(java.lang.String, java.util.ArrayList, java.lang.String)
 290          */
 291         @Override
 292         public CachedRowSet getData(String statement, ArrayList<String> arguments, String preferredDS) throws SQLException {
 293                 ArrayList<Object> newList=new ArrayList<Object>();
 294                 if(arguments != null && !arguments.isEmpty()) {
 295                         newList.addAll(arguments);
 296                 }
 297                 if(recoveryMode)
 298                         return requestDataWithRecovery(statement, newList, preferredDS);
 299                 else
 300                         return requestDataNoRecovery(statement, newList, preferredDS);
 301         }
 302
 303         private CachedRowSet requestDataWithRecovery(String statement, ArrayList<Object> arguments, String preferredDS) throws SQLException {
 304                 Throwable lastException = null;
 305                 CachedDataSource active = null;
 306
 307                 // test if there are any connection pools available
 308                 LinkedList<CachedDataSource> sources = new LinkedList<CachedDataSource>(this.dsQueue);
 309                 if(sources.isEmpty()){
 310                         LOGGER.error("Generated alarm: DBResourceManager.getData - No active DB connection pools are available.");
 311                         throw new DBLibException("No active DB connection pools are available in RequestDataWithRecovery call.");
 312                 }
 313                 if(preferredDS != null && !sources.peek().getDbConnectionName().equals(preferredDS)) {
 314                         Collections.reverse(sources);
 315                 }
 316
 317
 318                 // loop through available data sources to retrieve data.
 319                 while(!sources.isEmpty())
 320                 {
 321                         active = sources.peek();
 322
 323                         long time = System.currentTimeMillis();
 324                         try {
 325                                 if(!active.isFabric()) {
 326                                         CachedDataSource master = findMaster();
 327                                         if(master != null) {
 328                                                 active = master;
 329                                                 master = null;
 330                                         }
 331                                 }
 332                                 sources.remove(active);
 333                                 return active.getData(statement, arguments);
 334                         } catch(SQLDataException exc){
 335                                 throw exc;
 336                         } catch(SQLSyntaxErrorException exc){
 337                                 throw exc;
 338                         } catch(SQLIntegrityConstraintViolationException exc){
 339                                 throw exc;
 340                         } catch(Throwable exc){
 341                                 lastException = exc;
 342                                 String message = exc.getMessage();
 343                                 if(message == null) {
 344                                         if(exc.getCause() != null) {
 345                                                 message = exc.getCause().getMessage();
 346                                         }
 347                                         if(message == null)
 348                                                 message = exc.getClass().getName();
 349                                 }
 350                                 LOGGER.error("Generated alarm: "+active.getDbConnectionName()+" - "+message);
 351                                 handleGetConnectionException(active, exc);
 352                         } finally {
 353                                 if(LOGGER.isDebugEnabled()){
 354                                         time = (System.currentTimeMillis() - time);
 355                                         LOGGER.debug("getData processing time : "+ active.getDbConnectionName()+"  "+time+" miliseconds.");
 356                                 }
 357                         }
 358                 }
 359                 if(lastException instanceof SQLException){
 360                         throw (SQLException)lastException;
 361                 }
 362                 // repackage the exception
 363                 // you are here because either you run out of available data sources
 364                 // or the last exception was not of SQLException type.
 365                 // repackage the exception
 366                 if(lastException == null) {
 367                         throw new DBLibException("The operation timed out while waiting to acquire a new connection." );
 368                 } else {
 369                         SQLException exception = new DBLibException(lastException.getMessage());
 370                         exception.setStackTrace(lastException.getStackTrace());
 371                         if(lastException.getCause() instanceof SQLException) {
 372                                 throw (SQLException)lastException.getCause();
 373                         }
 374                         throw exception;
 375                 }
 376         }
 377
 378         private CachedRowSet requestDataNoRecovery(String statement, ArrayList<Object> arguments, String preferredDS) throws SQLException {
 379                 if(dsQueue.isEmpty()){
 380                         LOGGER.error("Generated alarm: DBResourceManager.getData - No active DB connection pools are available.");
 381                         throw new DBLibException("No active DB connection pools are available in RequestDataNoRecovery call.");
 382                 }
 383                 CachedDataSource active = (CachedDataSource) this.dsQueue.peek();
 384                 long time = System.currentTimeMillis();
 385                 try {
 386                         if(!active.isFabric()) {
 387                                 CachedDataSource master = findMaster();
 388                                 if(master != null)
 389                                         active = master;
 390                         }
 391                         return active.getData(statement, arguments);
 392 //              } catch(SQLDataException exc){
 393 //                      throw exc;
 394                 } catch(Throwable exc){
 395                         String message = exc.getMessage();
 396                         if(message == null)
 397                                 message = exc.getClass().getName();
 398                         LOGGER.error("Generated alarm: "+active.getDbConnectionName()+" - "+message);
 399                         if(exc instanceof SQLException)
 400                                 throw (SQLException)exc;
 401                         else {
 402                                 DBLibException excptn = new DBLibException(exc.getMessage());
 403                                 excptn.setStackTrace(exc.getStackTrace());
 404                                 throw excptn;
 405                         }
 406                 } finally {
 407                         if(LOGGER.isDebugEnabled()){
 408                                 time = (System.currentTimeMillis() - time);
 409                                 LOGGER.debug(">> getData : "+ active.getDbConnectionName()+"  "+time+" miliseconds.");
 410                         }
 411                 }
 412         }
 413
 414
 415         /* (non-Javadoc)
 416          * @see org.onap.ccsdk.sli.resource.dblib.DbLibService#writeData(java.lang.String, java.util.ArrayList, java.lang.String)
 417          */
 418         @Override
 419         public boolean writeData(String statement, ArrayList<String> arguments, String preferredDS) throws SQLException
 420         {
 421                 ArrayList<Object> newList=new ArrayList<Object>();
 422                 if(arguments != null && !arguments.isEmpty()) {
 423                         newList.addAll(arguments);
 424                 }
 425
 426                 return writeDataNoRecovery(statement, newList, preferredDS);
 427         }
 428
 429         CachedDataSource findMaster() throws PoolExhaustedException, MySQLNonTransientConnectionException {
 430                 CachedDataSource master = null;
 431                 CachedDataSource[] dss = this.dsQueue.toArray(new CachedDataSource[0]);
 432                 for(int i=0; i<dss.length; i++) {
 433                         if(!dss[i].isSlave()) {
 434                                 master = dss[i];
 435                                 if(i != 0) {
 436                                         dsQueue.remove(master);
 437                                         dsQueue.add(master);
 438                                 }
 439                                 return master;
 440                         }
 441                 }
 442                 if(master == null) {
 443                         LOGGER.warn("MASTER not found.");
 444                 }
 445                 return master;
 446         }
 447
 448
 449         private boolean writeDataNoRecovery(String statement, ArrayList<Object> arguments, String preferredDS) throws SQLException {
 450                 if(dsQueue.isEmpty()){
 451                         LOGGER.error("Generated alarm: DBResourceManager.getData - No active DB connection pools are available.");
 452                         throw new DBLibException("No active DB connection pools are available in RequestDataNoRecovery call.");
 453                 }
 454
 455                 boolean initialRequest = true;
 456                 boolean retryAllowed = true;
 457                 CachedDataSource active = (CachedDataSource) this.dsQueue.peek();
 458                 long time = System.currentTimeMillis();
 459                 while(initialRequest) {
 460                         initialRequest = false;
 461                         try {
 462                                 if(!active.isFabric()) {
 463                                         CachedDataSource master = findMaster();
 464                                         if(master != null) {
 465                                                 active = master;
 466                                         }
 467                                 }
 468
 469                                 return active.writeData(statement, arguments);
 470                         } catch(Throwable exc){
 471                                 String message = exc.getMessage();
 472                                 if(message == null)
 473                                         message = exc.getClass().getName();
 474                                 LOGGER.error("Generated alarm: "+active.getDbConnectionName()+" - "+message);
 475                                 if(exc instanceof SQLException) {
 476                                         SQLException sqlExc = SQLException.class.cast(exc);
 477                                         // handle read-only exception
 478                                         if(sqlExc.getErrorCode() == 1290 && "HY000".equals(sqlExc.getSQLState())) {
 479                                                 LOGGER.warn("retrying due to: " + sqlExc.getMessage());
 480                                                 dsQueue.remove(active);
 481                                                 dsQueue.add(active);
 482                                                 if(retryAllowed){
 483                                                         retryAllowed = false;
 484                                                         initialRequest = true;
 485                                                         continue;
 486                                                 }
 487                                         }
 488                                         throw (SQLException)exc;
 489                                 } else {
 490                                         DBLibException excptn = new DBLibException(exc.getMessage());
 491                                         excptn.setStackTrace(exc.getStackTrace());
 492                                         throw excptn;
 493                                 }
 494                         } finally {
 495                                 if(LOGGER.isDebugEnabled()){
 496                                         time = (System.currentTimeMillis() - time);
 497                                         LOGGER.debug("writeData processing time : "+ active.getDbConnectionName()+"  "+time+" miliseconds.");
 498                                 }
 499                         }
 500                 }
 501                 return true;
 502         }
 503
 504         private void setDataSource(CachedDataSource dataSource) {
 505                 if(dataSource.testConnection(true)){
 506                         this.dsQueue.add(dataSource);
 507                 } else {
 508                         this.broken.add(dataSource);
 509                 }
 510         }
 511
 512         public Connection getConnection() throws SQLException {
 513                 Throwable lastException = null;
 514                 CachedDataSource active = null;
 515
 516                 if(dsQueue.isEmpty()){
 517                         throw new DBLibException("No active DB connection pools are available in GetConnection call.");
 518                 }
 519
 520                 try {
 521                         active = dsQueue.peek();
 522                         CachedDataSource tmpActive = findMaster();
 523                         if(tmpActive != null) {
 524                                 active = tmpActive;
 525                         }
 526                         return new DBLibConnection(active.getConnection(), active);
 527                 } catch(javax.sql.rowset.spi.SyncFactoryException exc){
 528                         LOGGER.debug("Free memory (bytes): " + Runtime.getRuntime().freeMemory());
 529                         LOGGER.warn("CLASSPATH issue. Allowing retry", exc);
 530                         lastException = exc;
 531                 } catch(PoolExhaustedException exc) {
 532                         throw new NoAvailableConnectionsException(exc);
 533                 } catch(MySQLNonTransientConnectionException exc){
 534                         throw new NoAvailableConnectionsException(exc);
 535                 } catch(Exception exc){
 536                         lastException = exc;
 537                         if(recoveryMode){
 538                                 handleGetConnectionException(active, exc);
 539                         } else {
 540                                 if(exc instanceof MySQLNonTransientConnectionException) {
 541                                         throw new NoAvailableConnectionsException(exc);
 542                                 } if(exc instanceof SQLException) {
 543                                         throw (SQLException)exc;
 544                                 } else {
 545                                         DBLibException excptn = new DBLibException(exc.getMessage());
 546                                         excptn.setStackTrace(exc.getStackTrace());
 547                                         throw excptn;
 548                                 }
 549                         }
 550                 } catch (Throwable trwb) {
 551                         DBLibException excptn = new DBLibException(trwb.getMessage());
 552                         excptn.setStackTrace(trwb.getStackTrace());
 553                         throw excptn;
 554                 } finally {
 555                         if(LOGGER.isDebugEnabled()){
 556                                 displayState();
 557                         }
 558                 }
 559
 560                 if(lastException instanceof SQLException){
 561                         throw (SQLException)lastException;
 562                 }
 563                 // repackage the exception
 564                 if(lastException == null) {
 565                         throw new DBLibException("The operation timed out while waiting to acquire a new connection." );
 566                 } else {
 567                         SQLException exception = new DBLibException(lastException.getMessage());
 568                         exception.setStackTrace(lastException.getStackTrace());
 569                         if(lastException.getCause() instanceof SQLException) {
 570 //                              exception.setNextException((SQLException)lastException.getCause());
 571                                 throw (SQLException)lastException.getCause();
 572                         }
 573                         throw exception;
 574                 }
 575         }
 576
 577         public Connection getConnection(String username, String password)
 578         throws SQLException {
 579                 CachedDataSource active = null;
 580
 581                 if(dsQueue.isEmpty()){
 582                         throw new DBLibException("No active DB connection pools are available in GetConnection call.");
 583                 }
 584
 585
 586                 try {
 587                         active = dsQueue.peek();
 588                         CachedDataSource tmpActive = findMaster();
 589                         if(tmpActive != null) {
 590                                 active = tmpActive;
 591                         }
 592                         return active.getConnection(username, password);
 593                 } catch(Throwable exc){
 594                         if(recoveryMode){
 595                                 handleGetConnectionException(active, exc);
 596                         } else {
 597                                 if(exc instanceof SQLException)
 598                                         throw (SQLException)exc;
 599                                 else {
 600                                         DBLibException excptn = new DBLibException(exc.getMessage());
 601                                         excptn.setStackTrace(exc.getStackTrace());
 602                                         throw excptn;
 603                                 }
 604                         }
 605
 606                 }
 607
 608                 throw new DBLibException("No connections available in DBResourceManager in GetConnection call.");
 609         }
 610
 611         private void handleGetConnectionException(CachedDataSource source, Throwable exc) {
 612                 try {
 613                         if(!source.canTakeOffLine())
 614                         {
 615                                 LOGGER.error("Could not switch due to blocking");
 616                                 return;
 617                         }
 618
 619                         boolean removed = dsQueue.remove(source);
 620                         if(!broken.contains(source))
 621                         {
 622                                 if(broken.add(source))
 623                                 {
 624                                         LOGGER.warn("DB Recovery: DataSource <" + source.getDbConnectionName()  + "> put in the recovery mode. Reason : " + exc.getMessage());
 625                                 } else {
 626                                         LOGGER.warn("Error putting DataSource <" +source.getDbConnectionName()+  "> in recovery mode.");
 627                                 }
 628                         } else {
 629                                 LOGGER.info("DB Recovery: DataSource <" + source.getDbConnectionName() + "> already in recovery queue");
 630                         }
 631                         if(removed)
 632                         {
 633                                 if(!dsQueue.isEmpty())
 634                                 {
 635                                         LOGGER.warn("DB DataSource <" + dsQueue.peek().getDbConnectionName()    + "> became active");
 636                                 }
 637                         }
 638                 } catch (Exception e) {
 639                         LOGGER.error("", e);
 640                 }
 641         }
 642
 643         public void cleanUp() {
 644                 for(Iterator<CachedDataSource> it=dsQueue.iterator();it.hasNext();){
 645                         CachedDataSource cds = (CachedDataSource)it.next();
 646                         it.remove();
 647                         cds.cleanUp();
 648                 }
 649
 650                 try {
 651                         this.terminating = true;
 652                         if(broken != null)
 653                         {
 654                                 try {
 655                                         broken.add( new TerminatingCachedDataSource(null));
 656                                 } catch(Exception exc){
 657                                         LOGGER.error("Waiting for Worker to stop", exc);
 658                                 }
 659                         }
 660                         worker.join(terminationTimeOut);
 661                         LOGGER.info("DBResourceManager.RecoveryMgr <"+worker.toString() +"> termination was successful: " + worker.getState());
 662                 } catch(Exception exc){
 663                         LOGGER.error("Waiting for Worker thread to terminate ", exc);
 664                 }
 665         }
 666
 667         public static DBResourceManager create(Properties props) throws Exception {
 668                 DBResourceManager dbmanager = new DBResourceManager(props);
 669                 dbmanager.config(props);
 670                 return dbmanager;
 671         }
 672
 673         public PrintWriter getLogWriter() throws SQLException {
 674                 return ((CachedDataSource)this.dsQueue.peek()).getLogWriter();
 675         }
 676
 677         public int getLoginTimeout() throws SQLException {
 678                 return ((CachedDataSource)this.dsQueue.peek()).getLoginTimeout();
 679         }
 680
 681         public void setLogWriter(PrintWriter out) throws SQLException {
 682                 ((CachedDataSource)this.dsQueue.peek()).setLogWriter(out);
 683         }
 684
 685         public void setLoginTimeout(int seconds) throws SQLException {
 686                 ((CachedDataSource)this.dsQueue.peek()).setLoginTimeout(seconds);
 687         }
 688
 689         public void displayState(){
 690                 if(LOGGER.isDebugEnabled()){
 691                         LOGGER.debug("POOLS : Active = "+dsQueue.size() + ";\t Broken = "+broken.size());
 692                         CachedDataSource current = (CachedDataSource)dsQueue.peek();
 693                         if(current != null) {
 694                                 LOGGER.debug("POOL : Active name = \'"+current.getDbConnectionName()+ "\'");
 695                         }
 696                 }
 697         }
 698
 699         /* (non-Javadoc)
 700          * @see org.onap.ccsdk.sli.resource.dblib.DbLibService#isActive()
 701          */
 702         @Override
 703         public boolean isActive() {
 704                 return this.dsQueue.size()>0;
 705         }
 706
 707         public String getActiveStatus(){
 708                 return "Connected: " + dsQueue.size()+"\tIn-recovery: "+broken.size();
 709         }
 710
 711         public String getDBStatus(boolean htmlFormat) {
 712                 StringBuilder buffer = new StringBuilder();
 713
 714                 ArrayList<CachedDataSource> list = new ArrayList<CachedDataSource>();
 715                 list.addAll(dsQueue);
 716                 list.addAll(broken);
 717                 if (htmlFormat)
 718                 {
 719                         buffer.append("<tr class=\"headerRow\"><th id=\"header1\">")
 720                                         .append("Name:").append("</th>");
 721                         for (int i = 0; i < list.size(); i++) {
 722                                 buffer.append("<th id=\"header").append(2 + i).append("\">");
 723                                 buffer.append(((CachedDataSource) list.get(i)).getDbConnectionName()).append("</th>");
 724                         }
 725                         buffer.append("</tr>");
 726
 727                         buffer.append("<tr><td>State:</td>");
 728                         for (int i = 0; i < list.size(); i++) {
 729                                 if (broken.contains(list.get(i))) {
 730                                         buffer.append("<td>in recovery</td>");
 731                                 }
 732                                 if (dsQueue.contains(list.get(i))) {
 733                                         if (dsQueue.peek() == list.get(i))
 734                                                 buffer.append("<td>active</td>");
 735                                         else
 736                                                 buffer.append("<td>standby</td>");
 737                                 }
 738                         }
 739                         buffer.append("</tr>");
 740
 741                 } else {
 742                         for (int i = 0; i < list.size(); i++) {
 743                                 buffer.append("Name: ").append(((CachedDataSource) list.get(i)).getDbConnectionName());
 744                                 buffer.append("\tState: ");
 745                                 if (broken.contains(list.get(i))) {
 746                                         buffer.append("in recovery");
 747                                 } else
 748                                 if (dsQueue.contains(list.get(i))) {
 749                                         if (dsQueue.peek() == list.get(i))
 750                                                 buffer.append("active");
 751                                         else
 752                                                 buffer.append("standby");
 753                                 }
 754
 755                                 buffer.append("\n");
 756
 757                         }
 758                 }
 759                 return buffer.toString();
 760         }
 761
 762         public boolean isWrapperFor(Class<?> iface) throws SQLException {
 763                 return false;
 764         }
 765
 766         public <T> T unwrap(Class<T> iface) throws SQLException {
 767                 return null;
 768         }
 769
 770         /**
 771          * @return the monitorDbResponse
 772          */
 773         public final boolean isMonitorDbResponse() {
 774                 return recoveryMode && monitorDbResponse;
 775         }
 776
 777         public void test(){
 778                 CachedDataSource obj = dsQueue.peek();
 779                 Exception ption = new Exception();
 780                 try {
 781                         for(int i=0; i<5; i++)
 782                         {
 783                                 handleGetConnectionException(obj, ption);
 784                         }
 785                 } catch(Throwable exc){
 786                         LOGGER.warn("", exc);
 787                 }
 788         }
 789
 790         public String getPreferredDSName(){
 791                 if(isActive()){
 792                         return getPreferredDataSourceName(dsSelector);
 793                 }
 794                 return "";
 795         }
 796
 797         public String getPreferredDataSourceName(AtomicBoolean flipper) {
 798
 799                 LinkedList<CachedDataSource> snapshot = new LinkedList<CachedDataSource>(dsQueue);
 800                 if(snapshot.size() > 1){
 801                         CachedDataSource first = snapshot.getFirst();
 802                         CachedDataSource last = snapshot.getLast();
 803
 804                         int delta = first.getMonitor().getPorcessedConnectionsCount() - last.getMonitor().getPorcessedConnectionsCount();
 805                         if(delta < 0) {
 806                                 flipper.set(false);
 807                         } else if(delta > 0) {
 808                                 flipper.set(true);
 809                         } else {
 810                                 // check the last value and return !last
 811                                 flipper.getAndSet(!flipper.get());
 812                         }
 813
 814                         if (flipper.get())
 815                                 Collections.reverse(snapshot);
 816                 }
 817                 return snapshot.peek().getDbConnectionName();
 818         }
 819
 820         public java.util.logging.Logger getParentLogger()
 821                         throws SQLFeatureNotSupportedException {
 822                 return null;
 823         }
 824
 825         public String getMasterName() {
 826                 if(isActive()){
 827                         return getMasterDataSourceName(dsSelector);
 828                 }
 829                 return "";
 830         }
 831
 832
 833         private String getMasterDataSourceName(AtomicBoolean flipper) {
 834
 835                 LinkedList<CachedDataSource> snapshot = new LinkedList<CachedDataSource>(dsQueue);
 836                 if(snapshot.size() > 1){
 837                         CachedDataSource first = snapshot.getFirst();
 838                         CachedDataSource last = snapshot.getLast();
 839
 840                         int delta = first.getMonitor().getPorcessedConnectionsCount() - last.getMonitor().getPorcessedConnectionsCount();
 841                         if(delta < 0) {
 842                                 flipper.set(false);
 843                         } else if(delta > 0) {
 844                                 flipper.set(true);
 845                         } else {
 846                                 // check the last value and return !last
 847                                 flipper.getAndSet(!flipper.get());
 848                         }
 849
 850                         if (flipper.get())
 851                                 Collections.reverse(snapshot);
 852                 }
 853                 return snapshot.peek().getDbConnectionName();
 854         }
 855
 856     class RemindTask extends TimerTask {
 857         public void run() {
 858                         CachedDataSource ds = dsQueue.peek();
 859                         if(ds != null)
 860                                 ds.getPoolInfo(false);
 861         }
 862     }
 863 }