dblib/provider/src/main/java/org/openecomp/sdnc/sli/resource/dblib/DBResourceManager.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * openecomp
   4  * ================================================================================
   5  * Copyright (C) 2016 - 2017 AT&T
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdnc.sli.resource.dblib;
  22
  23 import java.io.PrintWriter;
  24 import java.sql.Connection;
  25 import java.sql.SQLDataException;
  26 import java.sql.SQLException;
  27 import java.sql.SQLFeatureNotSupportedException;
  28 import java.sql.SQLIntegrityConstraintViolationException;
  29 import java.sql.SQLSyntaxErrorException;
  30 import java.util.ArrayList;
  31 import java.util.Collections;
  32 import java.util.Comparator;
  33 import java.util.HashSet;
  34 import java.util.Iterator;
  35 import java.util.LinkedList;
  36 import java.util.Observable;
  37 import java.util.PriorityQueue;
  38 import java.util.Properties;
  39 import java.util.Queue;
  40 import java.util.Set;
  41 import java.util.Timer;
  42 import java.util.TimerTask;
  43 import java.util.concurrent.ConcurrentLinkedQueue;
  44 import java.util.concurrent.atomic.AtomicBoolean;
  45
  46 import javax.sql.DataSource;
  47 import javax.sql.rowset.CachedRowSet;
  48
  49 import org.apache.tomcat.jdbc.pool.PoolExhaustedException;
  50 import org.openecomp.sdnc.sli.resource.dblib.config.DbConfigPool;
  51 import org.openecomp.sdnc.sli.resource.dblib.factory.AbstractDBResourceManagerFactory;
  52 import org.openecomp.sdnc.sli.resource.dblib.factory.AbstractResourceManagerFactory;
  53 import org.openecomp.sdnc.sli.resource.dblib.factory.DBConfigFactory;
  54 import org.openecomp.sdnc.sli.resource.dblib.pm.PollingWorker;
  55 import org.openecomp.sdnc.sli.resource.dblib.pm.SQLExecutionMonitor;
  56 import com.mysql.jdbc.exceptions.jdbc4.MySQLNonTransientConnectionException;
  57 import org.slf4j.Logger;
  58 import org.slf4j.LoggerFactory;
  59
  60
  61 /**
  62  * @version $Revision: 1.15 $
  63  * Change Log
  64  * Author         Date     Comments
  65  * ============== ======== ====================================================
  66  * Rich Tabedzki
  67  */
  68 public class DBResourceManager implements DataSource, DataAccessor, DBResourceObserver, DbLibService {
  69         private static Logger LOGGER = LoggerFactory.getLogger(DBResourceManager.class);
  70
  71         transient boolean terminating = false;
  72         transient protected long retryInterval = 10000L;
  73         transient boolean recoveryMode = true;
  74
  75         protected final AtomicBoolean dsSelector = new  AtomicBoolean();
  76
  77 //      Queue<CachedDataSource> dsQueue = new ConcurrentLinkedQueue<CachedDataSource>();
  78         Queue<CachedDataSource> dsQueue = new PriorityQueue<CachedDataSource>(4, new Comparator<CachedDataSource>(){
  79
  80                 @Override
  81                 public int compare(CachedDataSource left, CachedDataSource right) {
  82                         try {
  83                                 if(!left.isSlave())
  84                                         return -1;
  85                                 if(!right.isSlave())
  86                                         return 1;
  87
  88                         } catch (Throwable e) {
  89                                 LOGGER.warn("", e);
  90                         }
  91                         return 0;
  92                 }
  93
  94         });
  95         protected final Set<CachedDataSource> broken = Collections.synchronizedSet(new HashSet<CachedDataSource>());
  96         protected final Object monitor = new Object();
  97         protected final Properties configProps;
  98         protected final Thread worker;
  99
 100         protected final long terminationTimeOut;
 101         protected final boolean monitorDbResponse;
 102         protected final long monitoringInterval;
 103         protected final long monitoringInitialDelay;
 104         protected final long expectedCompletionTime;
 105         protected final long unprocessedFailoverThreshold;
 106
 107         public DBResourceManager(Properties props){
 108                 this.configProps = props;
 109
 110                 // get retry interval value
 111                 retryInterval = getLongFromProperties(props, "org.openecomp.dblib.connection.retry", 10000L);
 112
 113                 // get recovery mode flag
 114                 recoveryMode = getBooleanFromProperties(props, "org.openecomp.dblib.connection.recovery", true);
 115                 if(!recoveryMode)
 116                 {
 117                         recoveryMode = false;
 118                         LOGGER.info("Recovery Mode disabled");
 119                 }
 120                 // get time out value for thread cleanup
 121                 terminationTimeOut = getLongFromProperties(props, "org.openecomp.dblib.termination.timeout", 300000L);
 122                 // get properties for monitoring
 123                 monitorDbResponse = getBooleanFromProperties(props, "org.openecomp.dblib.connection.monitor", false);
 124                 monitoringInterval = getLongFromProperties(props, "org.openecomp.dblib.connection.monitor.interval", 1000L);
 125                 monitoringInitialDelay = getLongFromProperties(props, "org.openecomp.dblib.connection.monitor.startdelay", 5000L);
 126                 expectedCompletionTime = getLongFromProperties(props, "org.openecomp.dblib.connection.monitor.expectedcompletiontime", 5000L);
 127                 unprocessedFailoverThreshold = getLongFromProperties(props, "org.openecomp.dblib.connection.monitor.unprocessedfailoverthreshold", 3L);
 128
 129                 // initialize performance monitor
 130                 PollingWorker.createInistance(props);
 131
 132                 // initialize recovery thread
 133                 worker = new RecoveryMgr();
 134                 worker.setName("DBResourcemanagerWatchThread");
 135                 worker.setDaemon(true);
 136                 worker.start();
 137         }
 138
 139         private void config(Properties ctx) throws Exception {
 140
 141                 DbConfigPool dbConfig = DBConfigFactory.createConfig(this.configProps);
 142
 143                 try {
 144                         AbstractResourceManagerFactory factory =  AbstractDBResourceManagerFactory.getFactory(dbConfig.getType());
 145                         if(LOGGER.isInfoEnabled()){
 146                                 LOGGER.info("Default DB config is : " + dbConfig.getType());
 147                                 LOGGER.info("Using factory : " + factory.getClass().getName());
 148                         }
 149                         CachedDataSource[] cachedDS = factory.initDBResourceManager(dbConfig, this);
 150                         if(cachedDS == null || cachedDS.length == 0) {
 151                                 LOGGER.error("Initialization of CachedDataSources failed. No instance was created.");
 152                                 throw new Exception("Failed to initialize DB Library. No data source was created.");
 153                         }
 154
 155                         for(int i=0; i<cachedDS.length; i++){
 156                                 if(cachedDS[i] != null && cachedDS[i].isInitialized()){
 157                                         setDataSource(cachedDS[i]);
 158                                         cachedDS[i].setInterval(monitoringInterval);
 159                                         cachedDS[i].setInitialDelay(monitoringInitialDelay);
 160                                         cachedDS[i].setExpectedCompletionTime(expectedCompletionTime);
 161                                         cachedDS[i].setUnprocessedFailoverThreshold(unprocessedFailoverThreshold);
 162                                         cachedDS[i].addObserver(this);
 163                                 }
 164                         }
 165
 166                 } catch(Exception exc){
 167                         throw exc;
 168                 }
 169         }
 170
 171         private long getLongFromProperties(Properties props, String property, long defaultValue)
 172         {
 173                 String value = null;
 174                 long tmpLongValue = defaultValue;
 175                 try {
 176                         value = (String)props.getProperty(property);
 177                         if(value != null)
 178                                 tmpLongValue = Long.parseLong(value);
 179
 180                 } catch(NumberFormatException exc) {
 181                         if(LOGGER.isWarnEnabled()){
 182                                 LOGGER.warn("'"+property+"'=" + value+" is invalid. It should be a numeric value");
 183                         }
 184                 } catch(Exception exc) {
 185                 }
 186                 return tmpLongValue;
 187
 188         }
 189
 190         private boolean getBooleanFromProperties(Properties props, String property, boolean defaultValue)
 191         {
 192                 boolean tmpValue = defaultValue;
 193                 String value = null;
 194
 195                 try {
 196                         value = (String)props.getProperty(property);
 197                         if(value != null)
 198                                 tmpValue = Boolean.parseBoolean(value);
 199
 200                 } catch(NumberFormatException exc) {
 201                         if(LOGGER.isWarnEnabled()){
 202                                 LOGGER.warn("'"+property+"'=" + value+" is invalid. It should be a boolean value");
 203                         }
 204                 } catch(Exception exc) {
 205                 }
 206                 return tmpValue;
 207
 208         }
 209
 210
 211         public void update(Observable observable, Object data) {
 212                 // if observable is active and there is a standby available, switch
 213                 if(observable instanceof SQLExecutionMonitor)
 214                 {
 215                         SQLExecutionMonitor monitor = (SQLExecutionMonitor)observable;
 216                         if(monitor.getParent() instanceof CachedDataSource)
 217                         {
 218                                 CachedDataSource dataSource = (CachedDataSource)monitor.getParent();
 219                                 if(dataSource == dsQueue.peek())
 220                                 {
 221                                         if(recoveryMode && dsQueue.size() > 1){
 222                                                 handleGetConnectionException(dataSource, new Exception(data.toString()));
 223                                         }
 224                                 }
 225                         }
 226                 }
 227         }
 228
 229         public void testForceRecovery()
 230         {
 231                 CachedDataSource active = (CachedDataSource) this.dsQueue.peek();
 232                 handleGetConnectionException(active, new Exception("test"));
 233         }
 234
 235         class RecoveryMgr extends Thread {
 236
 237                 public void run() {
 238                         while(!terminating)
 239                         {
 240                                 try {
 241                                         Thread.sleep(retryInterval);
 242                                 } catch (InterruptedException e1) {     }
 243                                 CachedDataSource brokenSource = null;
 244                                 try {
 245                                         if (!broken.isEmpty()) {
 246                                                 CachedDataSource[] sourceArray = broken.toArray(new CachedDataSource[0]);
 247                                                 for (int i = 0; i < sourceArray.length; i++)
 248                                                 {
 249                                                         brokenSource = sourceArray[i];
 250                                                         if (brokenSource instanceof TerminatingCachedDataSource)
 251                                                                 break;
 252                                                         if (resetConnectionPool(brokenSource)) {
 253                                                                 broken.remove(brokenSource);
 254                                                                 brokenSource.blockImmediateOffLine();
 255                                                                 dsQueue.add(brokenSource);
 256                                                                 LOGGER.info("DataSource <"
 257                                                                                 + brokenSource.getDbConnectionName()
 258                                                                                 + "> recovered.");
 259                                                         }
 260                                                         brokenSource = null;
 261                                                 }
 262                                         }
 263                                 } catch (Exception exc) {
 264                                         LOGGER.warn(exc.getMessage());
 265                                         if(brokenSource != null){
 266                                                 try {
 267                                                         if(!broken.contains(brokenSource))
 268                                                                 broken.add(brokenSource);
 269                                                         brokenSource = null;
 270                                                 } catch (Exception e1) {        }
 271                                         }
 272                                 }
 273                         }
 274                         LOGGER.info("DBResourceManager.RecoveryMgr <"+this.toString() +"> terminated." );
 275                 }
 276
 277                 private boolean resetConnectionPool(CachedDataSource dataSource){
 278                         try {
 279                                 return dataSource.testConnection();
 280                         } catch (Exception exc) {
 281                                 LOGGER.info("DataSource <" + dataSource.getDbConnectionName() + "> resetCache failed with error: "+ exc.getMessage());
 282                                 return false;
 283                         }
 284                 }
 285         }
 286
 287         /* (non-Javadoc)
 288          * @see org.openecomp.sdnc.sli.resource.dblib.DbLibService#getData(java.lang.String, java.util.ArrayList, java.lang.String)
 289          */
 290         @Override
 291         public CachedRowSet getData(String statement, ArrayList<String> arguments, String preferredDS) throws SQLException {
 292                 ArrayList<Object> newList=new ArrayList<Object>();
 293                 if(arguments != null && !arguments.isEmpty()) {
 294                         newList.addAll(arguments);
 295                 }
 296                 if(recoveryMode)
 297                         return requestDataWithRecovery(statement, newList, preferredDS);
 298                 else
 299                         return requestDataNoRecovery(statement, newList, preferredDS);
 300         }
 301
 302         private CachedRowSet requestDataWithRecovery(String statement, ArrayList<Object> arguments, String preferredDS) throws SQLException {
 303                 Throwable lastException = null;
 304                 CachedDataSource active = null;
 305
 306                 // test if there are any connection pools available
 307                 LinkedList<CachedDataSource> sources = new LinkedList<CachedDataSource>(this.dsQueue);
 308                 if(sources.isEmpty()){
 309                         LOGGER.error("Generated alarm: DBResourceManager.getData - No active DB connection pools are available.");
 310                         throw new DBLibException("No active DB connection pools are available in RequestDataWithRecovery call.");
 311                 }
 312                 if(preferredDS != null && !sources.peek().getDbConnectionName().equals(preferredDS)) {
 313                         Collections.reverse(sources);
 314                 }
 315
 316
 317                 // loop through available data sources to retrieve data.
 318                 while(!sources.isEmpty())
 319                 {
 320                         active = sources.peek();
 321
 322                         long time = System.currentTimeMillis();
 323                         try {
 324                                 if(!active.isFabric()) {
 325                                         CachedDataSource master = findMaster();
 326                                         if(master != null) {
 327                                                 active = master;
 328                                                 master = null;
 329                                         }
 330                                 }
 331                                 sources.remove(active);
 332                                 return active.getData(statement, arguments);
 333                         } catch(SQLDataException exc){
 334                                 throw exc;
 335                         } catch(SQLSyntaxErrorException exc){
 336                                 throw exc;
 337                         } catch(SQLIntegrityConstraintViolationException exc){
 338                                 throw exc;
 339                         } catch(Throwable exc){
 340                                 lastException = exc;
 341                                 String message = exc.getMessage();
 342                                 if(message == null) {
 343                                         if(exc.getCause() != null) {
 344                                                 message = exc.getCause().getMessage();
 345                                         }
 346                                         if(message == null)
 347                                                 message = exc.getClass().getName();
 348                                 }
 349                                 LOGGER.error("Generated alarm: "+active.getDbConnectionName()+" - "+message);
 350                                 handleGetConnectionException(active, exc);
 351                         } finally {
 352                                 if(LOGGER.isDebugEnabled()){
 353                                         time = (System.currentTimeMillis() - time);
 354                                         LOGGER.debug("getData processing time : "+ active.getDbConnectionName()+"  "+time+" miliseconds.");
 355                                 }
 356                         }
 357                 }
 358                 if(lastException instanceof SQLException){
 359                         throw (SQLException)lastException;
 360                 }
 361                 // repackage the exception
 362                 // you are here because either you run out of available data sources
 363                 // or the last exception was not of SQLException type.
 364                 // repackage the exception
 365                 if(lastException == null) {
 366                         throw new DBLibException("The operation timed out while waiting to acquire a new connection." );
 367                 } else {
 368                         SQLException exception = new DBLibException(lastException.getMessage());
 369                         exception.setStackTrace(lastException.getStackTrace());
 370                         if(lastException.getCause() instanceof SQLException) {
 371                                 throw (SQLException)lastException.getCause();
 372                         }
 373                         throw exception;
 374                 }
 375         }
 376
 377         private CachedRowSet requestDataNoRecovery(String statement, ArrayList<Object> arguments, String preferredDS) throws SQLException {
 378                 if(dsQueue.isEmpty()){
 379                         LOGGER.error("Generated alarm: DBResourceManager.getData - No active DB connection pools are available.");
 380                         throw new DBLibException("No active DB connection pools are available in RequestDataNoRecovery call.");
 381                 }
 382                 CachedDataSource active = (CachedDataSource) this.dsQueue.peek();
 383                 long time = System.currentTimeMillis();
 384                 try {
 385                         if(!active.isFabric()) {
 386                                 CachedDataSource master = findMaster();
 387                                 if(master != null)
 388                                         active = master;
 389                         }
 390                         return active.getData(statement, arguments);
 391 //              } catch(SQLDataException exc){
 392 //                      throw exc;
 393                 } catch(Throwable exc){
 394                         String message = exc.getMessage();
 395                         if(message == null)
 396                                 message = exc.getClass().getName();
 397                         LOGGER.error("Generated alarm: "+active.getDbConnectionName()+" - "+message);
 398                         if(exc instanceof SQLException)
 399                                 throw (SQLException)exc;
 400                         else {
 401                                 DBLibException excptn = new DBLibException(exc.getMessage());
 402                                 excptn.setStackTrace(exc.getStackTrace());
 403                                 throw excptn;
 404                         }
 405                 } finally {
 406                         if(LOGGER.isDebugEnabled()){
 407                                 time = (System.currentTimeMillis() - time);
 408                                 LOGGER.debug(">> getData : "+ active.getDbConnectionName()+"  "+time+" miliseconds.");
 409                         }
 410                 }
 411         }
 412
 413
 414         /* (non-Javadoc)
 415          * @see org.openecomp.sdnc.sli.resource.dblib.DbLibService#writeData(java.lang.String, java.util.ArrayList, java.lang.String)
 416          */
 417         @Override
 418         public boolean writeData(String statement, ArrayList<String> arguments, String preferredDS) throws SQLException
 419         {
 420                 ArrayList<Object> newList=new ArrayList<Object>();
 421                 if(arguments != null && !arguments.isEmpty()) {
 422                         newList.addAll(arguments);
 423                 }
 424
 425                 return writeDataNoRecovery(statement, newList, preferredDS);
 426         }
 427
 428         CachedDataSource findMaster() throws PoolExhaustedException, MySQLNonTransientConnectionException {
 429                 CachedDataSource master = null;
 430                 CachedDataSource[] dss = this.dsQueue.toArray(new CachedDataSource[0]);
 431                 for(int i=0; i<dss.length; i++) {
 432                         if(!dss[i].isSlave()) {
 433                                 master = dss[i];
 434                                 if(i != 0) {
 435                                         dsQueue.remove(master);
 436                                         dsQueue.add(master);
 437                                 }
 438                                 return master;
 439                         }
 440                 }
 441                 if(master == null) {
 442                         LOGGER.warn("MASTER not found.");
 443                 }
 444                 return master;
 445         }
 446
 447
 448         private boolean writeDataNoRecovery(String statement, ArrayList<Object> arguments, String preferredDS) throws SQLException {
 449                 if(dsQueue.isEmpty()){
 450                         LOGGER.error("Generated alarm: DBResourceManager.getData - No active DB connection pools are available.");
 451                         throw new DBLibException("No active DB connection pools are available in RequestDataNoRecovery call.");
 452                 }
 453
 454                 boolean initialRequest = true;
 455                 boolean retryAllowed = true;
 456                 CachedDataSource active = (CachedDataSource) this.dsQueue.peek();
 457                 long time = System.currentTimeMillis();
 458                 while(initialRequest) {
 459                         initialRequest = false;
 460                         try {
 461                                 if(!active.isFabric()) {
 462                                         CachedDataSource master = findMaster();
 463                                         if(master != null) {
 464                                                 active = master;
 465                                         }
 466                                 }
 467
 468                                 return active.writeData(statement, arguments);
 469                         } catch(Throwable exc){
 470                                 String message = exc.getMessage();
 471                                 if(message == null)
 472                                         message = exc.getClass().getName();
 473                                 LOGGER.error("Generated alarm: "+active.getDbConnectionName()+" - "+message);
 474                                 if(exc instanceof SQLException) {
 475                                         SQLException sqlExc = SQLException.class.cast(exc);
 476                                         // handle read-only exception
 477                                         if(sqlExc.getErrorCode() == 1290 && "HY000".equals(sqlExc.getSQLState())) {
 478                                                 LOGGER.warn("retrying due to: " + sqlExc.getMessage());
 479                                                 dsQueue.remove(active);
 480                                                 dsQueue.add(active);
 481                                                 if(retryAllowed){
 482                                                         retryAllowed = false;
 483                                                         initialRequest = true;
 484                                                         continue;
 485                                                 }
 486                                         }
 487                                         throw (SQLException)exc;
 488                                 } else {
 489                                         DBLibException excptn = new DBLibException(exc.getMessage());
 490                                         excptn.setStackTrace(exc.getStackTrace());
 491                                         throw excptn;
 492                                 }
 493                         } finally {
 494                                 if(LOGGER.isDebugEnabled()){
 495                                         time = (System.currentTimeMillis() - time);
 496                                         LOGGER.debug("writeData processing time : "+ active.getDbConnectionName()+"  "+time+" miliseconds.");
 497                                 }
 498                         }
 499                 }
 500                 return true;
 501         }
 502
 503         private void setDataSource(CachedDataSource dataSource) {
 504                 if(dataSource.testConnection(true)){
 505                         this.dsQueue.add(dataSource);
 506                 } else {
 507                         this.broken.add(dataSource);
 508                 }
 509         }
 510
 511         public Connection getConnection() throws SQLException {
 512                 Throwable lastException = null;
 513                 CachedDataSource active = null;
 514
 515                 if(dsQueue.isEmpty()){
 516                         throw new DBLibException("No active DB connection pools are available in GetConnection call.");
 517                 }
 518
 519                 try {
 520                         active = dsQueue.peek();
 521                         CachedDataSource tmpActive = findMaster();
 522                         if(tmpActive != null) {
 523                                 active = tmpActive;
 524                         }
 525                         return new DBLibConnection(active.getConnection(), active);
 526                 } catch(javax.sql.rowset.spi.SyncFactoryException exc){
 527                         LOGGER.debug("Free memory (bytes): " + Runtime.getRuntime().freeMemory());
 528                         LOGGER.warn("CLASSPATH issue. Allowing retry", exc);
 529                         lastException = exc;
 530                 } catch(PoolExhaustedException exc) {
 531                         throw new NoAvailableConnectionsException(exc);
 532                 } catch(MySQLNonTransientConnectionException exc){
 533                         throw new NoAvailableConnectionsException(exc);
 534                 } catch(Exception exc){
 535                         lastException = exc;
 536                         if(recoveryMode){
 537                                 handleGetConnectionException(active, exc);
 538                         } else {
 539                                 if(exc instanceof MySQLNonTransientConnectionException) {
 540                                         throw new NoAvailableConnectionsException(exc);
 541                                 } if(exc instanceof SQLException) {
 542                                         throw (SQLException)exc;
 543                                 } else {
 544                                         DBLibException excptn = new DBLibException(exc.getMessage());
 545                                         excptn.setStackTrace(exc.getStackTrace());
 546                                         throw excptn;
 547                                 }
 548                         }
 549                 } catch (Throwable trwb) {
 550                         DBLibException excptn = new DBLibException(trwb.getMessage());
 551                         excptn.setStackTrace(trwb.getStackTrace());
 552                         throw excptn;
 553                 } finally {
 554                         if(LOGGER.isDebugEnabled()){
 555                                 displayState();
 556                         }
 557                 }
 558
 559                 if(lastException instanceof SQLException){
 560                         throw (SQLException)lastException;
 561                 }
 562                 // repackage the exception
 563                 if(lastException == null) {
 564                         throw new DBLibException("The operation timed out while waiting to acquire a new connection." );
 565                 } else {
 566                         SQLException exception = new DBLibException(lastException.getMessage());
 567                         exception.setStackTrace(lastException.getStackTrace());
 568                         if(lastException.getCause() instanceof SQLException) {
 569 //                              exception.setNextException((SQLException)lastException.getCause());
 570                                 throw (SQLException)lastException.getCause();
 571                         }
 572                         throw exception;
 573                 }
 574         }
 575
 576         public Connection getConnection(String username, String password)
 577         throws SQLException {
 578                 CachedDataSource active = null;
 579
 580                 if(dsQueue.isEmpty()){
 581                         throw new DBLibException("No active DB connection pools are available in GetConnection call.");
 582                 }
 583
 584
 585                 try {
 586                         active = dsQueue.peek();
 587                         CachedDataSource tmpActive = findMaster();
 588                         if(tmpActive != null) {
 589                                 active = tmpActive;
 590                         }
 591                         return active.getConnection(username, password);
 592                 } catch(Throwable exc){
 593                         if(recoveryMode){
 594                                 handleGetConnectionException(active, exc);
 595                         } else {
 596                                 if(exc instanceof SQLException)
 597                                         throw (SQLException)exc;
 598                                 else {
 599                                         DBLibException excptn = new DBLibException(exc.getMessage());
 600                                         excptn.setStackTrace(exc.getStackTrace());
 601                                         throw excptn;
 602                                 }
 603                         }
 604
 605                 }
 606
 607                 throw new DBLibException("No connections available in DBResourceManager in GetConnection call.");
 608         }
 609
 610         private void handleGetConnectionException(CachedDataSource source, Throwable exc) {
 611                 try {
 612                         if(!source.canTakeOffLine())
 613                         {
 614                                 LOGGER.error("Could not switch due to blocking");
 615                                 return;
 616                         }
 617
 618                         boolean removed = dsQueue.remove(source);
 619                         if(!broken.contains(source))
 620                         {
 621                                 if(broken.add(source))
 622                                 {
 623                                         LOGGER.warn("DB Recovery: DataSource <" + source.getDbConnectionName()  + "> put in the recovery mode. Reason : " + exc.getMessage());
 624                                 } else {
 625                                         LOGGER.warn("Error putting DataSource <" +source.getDbConnectionName()+  "> in recovery mode.");
 626                                 }
 627                         } else {
 628                                 LOGGER.info("DB Recovery: DataSource <" + source.getDbConnectionName() + "> already in recovery queue");
 629                         }
 630                         if(removed)
 631                         {
 632                                 if(!dsQueue.isEmpty())
 633                                 {
 634                                         LOGGER.warn("DB DataSource <" + dsQueue.peek().getDbConnectionName()    + "> became active");
 635                                 }
 636                         }
 637                 } catch (Exception e) {
 638                         LOGGER.error("", e);
 639                 }
 640         }
 641
 642         public void cleanUp() {
 643                 for(Iterator<CachedDataSource> it=dsQueue.iterator();it.hasNext();){
 644                         CachedDataSource cds = (CachedDataSource)it.next();
 645                         it.remove();
 646                         cds.cleanUp();
 647                 }
 648
 649                 try {
 650                         this.terminating = true;
 651                         if(broken != null)
 652                         {
 653                                 try {
 654                                         broken.add( new TerminatingCachedDataSource(null));
 655                                 } catch(Exception exc){
 656                                         LOGGER.error("Waiting for Worker to stop", exc);
 657                                 }
 658                         }
 659                         worker.join(terminationTimeOut);
 660                         LOGGER.info("DBResourceManager.RecoveryMgr <"+worker.toString() +"> termination was successful: " + worker.getState());
 661                 } catch(Exception exc){
 662                         LOGGER.error("Waiting for Worker thread to terminate ", exc);
 663                 }
 664         }
 665
 666         public static DBResourceManager create(Properties props) throws Exception {
 667                 DBResourceManager dbmanager = new DBResourceManager(props);
 668                 dbmanager.config(props);
 669                 return dbmanager;
 670         }
 671
 672         public PrintWriter getLogWriter() throws SQLException {
 673                 return ((CachedDataSource)this.dsQueue.peek()).getLogWriter();
 674         }
 675
 676         public int getLoginTimeout() throws SQLException {
 677                 return ((CachedDataSource)this.dsQueue.peek()).getLoginTimeout();
 678         }
 679
 680         public void setLogWriter(PrintWriter out) throws SQLException {
 681                 ((CachedDataSource)this.dsQueue.peek()).setLogWriter(out);
 682         }
 683
 684         public void setLoginTimeout(int seconds) throws SQLException {
 685                 ((CachedDataSource)this.dsQueue.peek()).setLoginTimeout(seconds);
 686         }
 687
 688         public void displayState(){
 689                 if(LOGGER.isDebugEnabled()){
 690                         LOGGER.debug("POOLS : Active = "+dsQueue.size() + ";\t Broken = "+broken.size());
 691                         CachedDataSource current = (CachedDataSource)dsQueue.peek();
 692                         if(current != null) {
 693                                 LOGGER.debug("POOL : Active name = \'"+current.getDbConnectionName()+ "\'");
 694                         }
 695                 }
 696         }
 697
 698         /* (non-Javadoc)
 699          * @see org.openecomp.sdnc.sli.resource.dblib.DbLibService#isActive()
 700          */
 701         @Override
 702         public boolean isActive() {
 703                 return this.dsQueue.size()>0;
 704         }
 705
 706         public String getActiveStatus(){
 707                 return "Connected: " + dsQueue.size()+"\tIn-recovery: "+broken.size();
 708         }
 709
 710         public String getDBStatus(boolean htmlFormat) {
 711                 StringBuilder buffer = new StringBuilder();
 712
 713                 ArrayList<CachedDataSource> list = new ArrayList<CachedDataSource>();
 714                 list.addAll(dsQueue);
 715                 list.addAll(broken);
 716                 if (htmlFormat)
 717                 {
 718                         buffer.append("<tr class=\"headerRow\"><th id=\"header1\">")
 719                                         .append("Name:").append("</th>");
 720                         for (int i = 0; i < list.size(); i++) {
 721                                 buffer.append("<th id=\"header").append(2 + i).append("\">");
 722                                 buffer.append(((CachedDataSource) list.get(i)).getDbConnectionName()).append("</th>");
 723                         }
 724                         buffer.append("</tr>");
 725
 726                         buffer.append("<tr><td>State:</td>");
 727                         for (int i = 0; i < list.size(); i++) {
 728                                 if (broken.contains(list.get(i))) {
 729                                         buffer.append("<td>in recovery</td>");
 730                                 }
 731                                 if (dsQueue.contains(list.get(i))) {
 732                                         if (dsQueue.peek() == list.get(i))
 733                                                 buffer.append("<td>active</td>");
 734                                         else
 735                                                 buffer.append("<td>standby</td>");
 736                                 }
 737                         }
 738                         buffer.append("</tr>");
 739
 740                 } else {
 741                         for (int i = 0; i < list.size(); i++) {
 742                                 buffer.append("Name: ").append(((CachedDataSource) list.get(i)).getDbConnectionName());
 743                                 buffer.append("\tState: ");
 744                                 if (broken.contains(list.get(i))) {
 745                                         buffer.append("in recovery");
 746                                 } else
 747                                 if (dsQueue.contains(list.get(i))) {
 748                                         if (dsQueue.peek() == list.get(i))
 749                                                 buffer.append("active");
 750                                         else
 751                                                 buffer.append("standby");
 752                                 }
 753
 754                                 buffer.append("\n");
 755
 756                         }
 757                 }
 758                 return buffer.toString();
 759         }
 760
 761         public boolean isWrapperFor(Class<?> iface) throws SQLException {
 762                 return false;
 763         }
 764
 765         public <T> T unwrap(Class<T> iface) throws SQLException {
 766                 return null;
 767         }
 768
 769         /**
 770          * @return the monitorDbResponse
 771          */
 772         public final boolean isMonitorDbResponse() {
 773                 return recoveryMode && monitorDbResponse;
 774         }
 775
 776         public void test(){
 777                 CachedDataSource obj = dsQueue.peek();
 778                 Exception ption = new Exception();
 779                 try {
 780                         for(int i=0; i<5; i++)
 781                         {
 782                                 handleGetConnectionException(obj, ption);
 783                         }
 784                 } catch(Throwable exc){
 785                         LOGGER.warn("", exc);
 786                 }
 787         }
 788
 789         public String getPreferredDSName(){
 790                 if(isActive()){
 791                         return getPreferredDataSourceName(dsSelector);
 792                 }
 793                 return "";
 794         }
 795
 796         public String getPreferredDataSourceName(AtomicBoolean flipper) {
 797
 798                 LinkedList<CachedDataSource> snapshot = new LinkedList<CachedDataSource>(dsQueue);
 799                 if(snapshot.size() > 1){
 800                         CachedDataSource first = snapshot.getFirst();
 801                         CachedDataSource last = snapshot.getLast();
 802
 803                         int delta = first.getMonitor().getPorcessedConnectionsCount() - last.getMonitor().getPorcessedConnectionsCount();
 804                         if(delta < 0) {
 805                                 flipper.set(false);
 806                         } else if(delta > 0) {
 807                                 flipper.set(true);
 808                         } else {
 809                                 // check the last value and return !last
 810                                 flipper.getAndSet(!flipper.get());
 811                         }
 812
 813                         if (flipper.get())
 814                                 Collections.reverse(snapshot);
 815                 }
 816                 return snapshot.peek().getDbConnectionName();
 817         }
 818
 819         public java.util.logging.Logger getParentLogger()
 820                         throws SQLFeatureNotSupportedException {
 821                 // TODO Auto-generated method stub
 822                 return null;
 823         }
 824
 825         public String getMasterName() {
 826                 if(isActive()){
 827                         return getMasterDataSourceName(dsSelector);
 828                 }
 829                 return "";
 830         }
 831
 832
 833         private String getMasterDataSourceName(AtomicBoolean flipper) {
 834
 835                 LinkedList<CachedDataSource> snapshot = new LinkedList<CachedDataSource>(dsQueue);
 836                 if(snapshot.size() > 1){
 837                         CachedDataSource first = snapshot.getFirst();
 838                         CachedDataSource last = snapshot.getLast();
 839
 840                         int delta = first.getMonitor().getPorcessedConnectionsCount() - last.getMonitor().getPorcessedConnectionsCount();
 841                         if(delta < 0) {
 842                                 flipper.set(false);
 843                         } else if(delta > 0) {
 844                                 flipper.set(true);
 845                         } else {
 846                                 // check the last value and return !last
 847                                 flipper.getAndSet(!flipper.get());
 848                         }
 849
 850                         if (flipper.get())
 851                                 Collections.reverse(snapshot);
 852                 }
 853                 return snapshot.peek().getDbConnectionName();
 854         }
 855
 856     class RemindTask extends TimerTask {
 857         public void run() {
 858                         CachedDataSource ds = dsQueue.peek();
 859                         if(ds != null)
 860                                 ds.getPoolInfo(false);
 861         }
 862     }
 863 }