dblib/provider/src/main/java/org/openecomp/sdnc/sli/resource/dblib/DBResourceManager.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * openecomp
   4  * ================================================================================
   5  * Copyright (C) 2016 - 2017 AT&T
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdnc.sli.resource.dblib;
  22
  23 import java.io.PrintWriter;
  24 import java.sql.Connection;
  25 import java.sql.SQLDataException;
  26 import java.sql.SQLException;
  27 import java.sql.SQLFeatureNotSupportedException;
  28 import java.sql.SQLIntegrityConstraintViolationException;
  29 import java.sql.SQLSyntaxErrorException;
  30 import java.util.ArrayList;
  31 import java.util.Collections;
  32 import java.util.Comparator;
  33 import java.util.HashSet;
  34 import java.util.Iterator;
  35 import java.util.LinkedList;
  36 import java.util.Observable;
  37 import java.util.PriorityQueue;
  38 import java.util.Properties;
  39 import java.util.Queue;
  40 import java.util.Set;
  41 import java.util.concurrent.ConcurrentLinkedQueue;
  42 import java.util.concurrent.atomic.AtomicBoolean;
  43
  44 import javax.sql.DataSource;
  45 import javax.sql.rowset.CachedRowSet;
  46
  47 import org.openecomp.sdnc.sli.resource.dblib.config.DbConfigPool;
  48 import org.openecomp.sdnc.sli.resource.dblib.factory.AbstractDBResourceManagerFactory;
  49 import org.openecomp.sdnc.sli.resource.dblib.factory.AbstractResourceManagerFactory;
  50 import org.openecomp.sdnc.sli.resource.dblib.factory.DBConfigFactory;
  51 import org.openecomp.sdnc.sli.resource.dblib.pm.PollingWorker;
  52 import org.openecomp.sdnc.sli.resource.dblib.pm.SQLExecutionMonitor;
  53 import org.slf4j.Logger;
  54 import org.slf4j.LoggerFactory;
  55
  56 import com.sun.rowset.providers.RIOptimisticProvider;
  57
  58 /**
  59  * @version $Revision: 1.15 $
  60  * Change Log
  61  * Author         Date     Comments
  62  * ============== ======== ====================================================
  63  * Rich Tabedzki
  64  */
  65 public class DBResourceManager implements DataSource, DataAccessor, DBResourceObserver, DbLibService {
  66         private static Logger LOGGER = LoggerFactory.getLogger(DBResourceManager.class);
  67
  68         transient boolean terminating = false;
  69         transient protected long retryInterval = 10000L;
  70         transient boolean recoveryMode = true;
  71
  72         protected final AtomicBoolean dsSelector = new  AtomicBoolean();
  73
  74 //      Queue<CachedDataSource> dsQueue = new ConcurrentLinkedQueue<CachedDataSource>();
  75         Queue<CachedDataSource> dsQueue = new PriorityQueue<CachedDataSource>(4, new Comparator<CachedDataSource>(){
  76
  77                 @Override
  78                 public int compare(CachedDataSource left, CachedDataSource right) {
  79                         try {
  80                                 if(!left.isSlave())
  81                                         return -1;
  82                                 if(!right.isSlave())
  83                                         return 1;
  84
  85                         } catch (Throwable e) {
  86                                 LOGGER.warn("", e);
  87                         }
  88                         return 0;
  89                 }
  90
  91         });
  92         protected final Set<CachedDataSource> broken = Collections.synchronizedSet(new HashSet<CachedDataSource>());
  93         protected final Object monitor = new Object();
  94         protected final Properties configProps;
  95         protected final Thread worker;
  96
  97         protected final long terminationTimeOut;
  98         protected final boolean monitorDbResponse;
  99         protected final long monitoringInterval;
 100         protected final long monitoringInitialDelay;
 101         protected final long expectedCompletionTime;
 102         protected final long unprocessedFailoverThreshold;
 103
 104         public DBResourceManager(Properties props){
 105                 this.configProps = props;
 106
 107                 // get retry interval value
 108                 retryInterval = getLongFromProperties(props, "org.openecomp.dblib.connection.retry", 10000L);
 109
 110                 // get recovery mode flag
 111                 recoveryMode = getBooleanFromProperties(props, "org.openecomp.dblib.connection.recovery", true);
 112                 if(!recoveryMode)
 113                 {
 114                         recoveryMode = false;
 115                         LOGGER.info("Recovery Mode disabled");
 116                 }
 117                 // get time out value for thread cleanup
 118                 terminationTimeOut = getLongFromProperties(props, "org.openecomp.dblib.termination.timeout", 300000L);
 119                 // get properties for monitoring
 120                 monitorDbResponse = getBooleanFromProperties(props, "org.openecomp.dblib.connection.monitor", false);
 121                 monitoringInterval = getLongFromProperties(props, "org.openecomp.dblib.connection.monitor.interval", 1000L);
 122                 monitoringInitialDelay = getLongFromProperties(props, "org.openecomp.dblib.connection.monitor.startdelay", 5000L);
 123                 expectedCompletionTime = getLongFromProperties(props, "org.openecomp.dblib.connection.monitor.expectedcompletiontime", 5000L);
 124                 unprocessedFailoverThreshold = getLongFromProperties(props, "org.openecomp.dblib.connection.monitor.unprocessedfailoverthreshold", 3L);
 125
 126                 // initialize performance monitor
 127                 PollingWorker.createInistance(props);
 128
 129                 // initialize recovery thread
 130                 worker = new RecoveryMgr();
 131                 worker.setName("DBResourcemanagerWatchThread");
 132                 worker.setDaemon(true);
 133                 worker.start();
 134
 135                 try {
 136                         RIOptimisticProvider rio  = new RIOptimisticProvider();
 137                         LOGGER.info("Class " + rio.getClass().getName() + " found.");
 138                         Class clas = this.getClass().getClassLoader().loadClass("com.sun.rowset.providers.RIOptimisticProvider");
 139                         if(clas != null) {
 140                                 LOGGER.info("Class " + rio.getClass().getName() + " found by classloader.");
 141                         }
 142                 } catch(Exception exc) {
 143                         LOGGER.info("Failed resolving RIOptimisticProvider class", exc);
 144                 }
 145         }
 146
 147         private void config(Properties ctx) throws Exception {
 148
 149                 DbConfigPool dbConfig = DBConfigFactory.createConfig(this.configProps);
 150
 151                 try {
 152                         AbstractResourceManagerFactory factory =  AbstractDBResourceManagerFactory.getFactory(dbConfig.getType());
 153                         if(LOGGER.isInfoEnabled()){
 154                                 LOGGER.info("Default DB config is : " + dbConfig.getType());
 155                                 LOGGER.info("Using factory : " + factory.getClass().getName());
 156                         }
 157                         CachedDataSource[] cachedDS = factory.initDBResourceManager(dbConfig, this);
 158                         if(cachedDS == null || cachedDS.length == 0) {
 159                                 LOGGER.error("Initialization of CachedDataSources failed. No instance was created.");
 160                                 throw new Exception("Failed to initialize DB Library. No data source was created.");
 161                         }
 162
 163                         for(int i=0; i<cachedDS.length; i++){
 164                                 if(cachedDS[i] != null && cachedDS[i].isInitialized()){
 165                                         setDataSource(cachedDS[i]);
 166                                         cachedDS[i].setInterval(monitoringInterval);
 167                                         cachedDS[i].setInitialDelay(monitoringInitialDelay);
 168                                         cachedDS[i].setExpectedCompletionTime(expectedCompletionTime);
 169                                         cachedDS[i].setUnprocessedFailoverThreshold(unprocessedFailoverThreshold);
 170                                         cachedDS[i].addObserver(this);
 171                                 }
 172                         }
 173
 174                 } catch(Exception exc){
 175                         throw exc;
 176                 }
 177         }
 178
 179         private long getLongFromProperties(Properties props, String property, long defaultValue)
 180         {
 181                 String value = null;
 182                 long tmpLongValue = defaultValue;
 183                 try {
 184                         value = (String)props.getProperty(property);
 185                         if(value != null)
 186                                 tmpLongValue = Long.parseLong(value);
 187
 188                 } catch(NumberFormatException exc) {
 189                         if(LOGGER.isWarnEnabled()){
 190                                 LOGGER.warn("'"+property+"'=" + value+" is invalid. It should be a numeric value");
 191                         }
 192                 } catch(Exception exc) {
 193                 }
 194                 return tmpLongValue;
 195
 196         }
 197
 198         private boolean getBooleanFromProperties(Properties props, String property, boolean defaultValue)
 199         {
 200                 boolean tmpValue = defaultValue;
 201                 String value = null;
 202
 203                 try {
 204                         value = (String)props.getProperty(property);
 205                         if(value != null)
 206                                 tmpValue = Boolean.parseBoolean(value);
 207
 208                 } catch(NumberFormatException exc) {
 209                         if(LOGGER.isWarnEnabled()){
 210                                 LOGGER.warn("'"+property+"'=" + value+" is invalid. It should be a boolean value");
 211                         }
 212                 } catch(Exception exc) {
 213                 }
 214                 return tmpValue;
 215
 216         }
 217
 218
 219         public void update(Observable observable, Object data) {
 220                 // if observable is active and there is a standby available, switch
 221                 if(observable instanceof SQLExecutionMonitor)
 222                 {
 223                         SQLExecutionMonitor monitor = (SQLExecutionMonitor)observable;
 224                         if(monitor.getParent() instanceof CachedDataSource)
 225                         {
 226                                 CachedDataSource dataSource = (CachedDataSource)monitor.getParent();
 227                                 if(dataSource == dsQueue.peek())
 228                                 {
 229                                         if(recoveryMode && dsQueue.size() > 1){
 230                                                 handleGetConnectionException(dataSource, new Exception(data.toString()));
 231                                         }
 232                                 }
 233                         }
 234                 }
 235         }
 236
 237         public void testForceRecovery()
 238         {
 239                 CachedDataSource active = (CachedDataSource) this.dsQueue.peek();
 240                 handleGetConnectionException(active, new Exception("test"));
 241         }
 242
 243         class RecoveryMgr extends Thread {
 244
 245                 public void run() {
 246                         while(!terminating)
 247                         {
 248                                 try {
 249                                         Thread.sleep(retryInterval);
 250                                 } catch (InterruptedException e1) {     }
 251                                 CachedDataSource brokenSource = null;
 252                                 try {
 253                                         if (!broken.isEmpty()) {
 254                                                 CachedDataSource[] sourceArray = broken.toArray(new CachedDataSource[0]);
 255                                                 for (int i = 0; i < sourceArray.length; i++)
 256                                                 {
 257                                                         brokenSource = sourceArray[i];
 258                                                         if (brokenSource instanceof TerminatingCachedDataSource)
 259                                                                 break;
 260                                                         if (resetConnectionPool(brokenSource)) {
 261                                                                 broken.remove(brokenSource);
 262                                                                 brokenSource.blockImmediateOffLine();
 263                                                                 dsQueue.add(brokenSource);
 264                                                                 LOGGER.info("DataSource <"
 265                                                                                 + brokenSource.getDbConnectionName()
 266                                                                                 + "> recovered.");
 267                                                         }
 268                                                         brokenSource = null;
 269                                                 }
 270                                         }
 271                                 } catch (Exception exc) {
 272                                         LOGGER.warn(exc.getMessage());
 273                                         if(brokenSource != null){
 274                                                 try {
 275                                                         if(!broken.contains(brokenSource))
 276                                                                 broken.add(brokenSource);
 277                                                         brokenSource = null;
 278                                                 } catch (Exception e1) {        }
 279                                         }
 280                                 }
 281                         }
 282                         LOGGER.info("DBResourceManager.RecoveryMgr <"+this.toString() +"> terminated." );
 283                 }
 284
 285                 private boolean resetConnectionPool(CachedDataSource dataSource){
 286                         try {
 287                                 return dataSource.testConnection();
 288                         } catch (Exception exc) {
 289                                 LOGGER.info("DataSource <" + dataSource.getDbConnectionName() + "> resetCache failed with error: "+ exc.getMessage());
 290                                 return false;
 291                         }
 292                 }
 293         }
 294
 295         /* (non-Javadoc)
 296          * @see org.openecomp.dblib.DataAccessor#getData(java.lang.String, java.util.ArrayList)
 297          */
 298         /* (non-Javadoc)
 299          * @see org.openecomp.sdnc.sli.resource.dblib.DbLibService#getData(java.lang.String, java.util.ArrayList, java.lang.String)
 300          */
 301         @Override
 302         public CachedRowSet getData(String statement, ArrayList<String> arguments, String preferredDS) throws SQLException {
 303                 if(recoveryMode)
 304                         return requestDataWithRecovery(statement, arguments, preferredDS);
 305                 else
 306                         return requestDataNoRecovery(statement, arguments, preferredDS);
 307         }
 308
 309         private CachedRowSet requestDataWithRecovery(String statement, ArrayList<String> arguments, String preferredDS) throws SQLException {
 310                 Throwable lastException = null;
 311                 CachedDataSource active = null;
 312
 313                 // test if there are any connection pools available
 314                 LinkedList<CachedDataSource> sources = new LinkedList<CachedDataSource>(this.dsQueue);
 315                 if(sources.isEmpty()){
 316                         LOGGER.error("Generated alarm: DBResourceManager.getData - No active DB connection pools are available.");
 317                         throw new DBLibException("No active DB connection pools are available in RequestDataWithRecovery call.");
 318                 }
 319                 if(preferredDS != null && !sources.peek().getDbConnectionName().equals(preferredDS)) {
 320                         Collections.reverse(sources);
 321                 }
 322
 323
 324                 // loop through available data sources to retrieve data.
 325                 while(!sources.isEmpty())
 326                 {
 327                         active = sources.peek();
 328
 329                         long time = System.currentTimeMillis();
 330                         try {
 331                                 if(!active.isFabric()) {
 332                                         CachedDataSource master = findMaster();
 333                                         if(master != null) {
 334                                                 active = master;
 335                                                 master = null;
 336                                         }
 337                                 }
 338                                 sources.remove(active);
 339                                 return active.getData(statement, arguments);
 340                         } catch(SQLDataException exc){
 341                                 throw exc;
 342                         } catch(SQLSyntaxErrorException exc){
 343                                 throw exc;
 344                         } catch(SQLIntegrityConstraintViolationException exc){
 345                                 throw exc;
 346                         } catch(Throwable exc){
 347                                 lastException = exc;
 348                                 String message = exc.getMessage();
 349                                 if(message == null) {
 350                                         if(exc.getCause() != null) {
 351                                                 message = exc.getCause().getMessage();
 352                                         }
 353                                         if(message == null)
 354                                                 message = exc.getClass().getName();
 355                                 }
 356                                 LOGGER.error("Generated alarm: "+active.getDbConnectionName()+" - "+message);
 357                                 handleGetConnectionException(active, exc);
 358                         } finally {
 359                                 if(LOGGER.isDebugEnabled()){
 360                                         time = (System.currentTimeMillis() - time);
 361                                         LOGGER.debug(">> getData : "+ active.getDbConnectionName()+"  "+time+" miliseconds.");
 362                                 }
 363                         }
 364                 }
 365                 if(lastException instanceof SQLException){
 366                         throw (SQLException)lastException;
 367                 }
 368                 // repackage the exception
 369                 // you are here because either you run out of available data sources
 370                 // or the last exception was not of SQLException type.
 371                 // repackage the exception
 372                 if(lastException == null) {
 373                         throw new DBLibException("The operation timed out while waiting to acquire a new connection." );
 374                 } else {
 375                         SQLException exception = new DBLibException(lastException.getMessage());
 376                         exception.setStackTrace(lastException.getStackTrace());
 377                         if(lastException.getCause() instanceof SQLException) {
 378                                 throw (SQLException)lastException.getCause();
 379                         }
 380                         throw exception;
 381                 }
 382         }
 383
 384         private CachedRowSet requestDataNoRecovery(String statement, ArrayList<String> arguments, String preferredDS) throws SQLException {
 385                 if(dsQueue.isEmpty()){
 386                         LOGGER.error("Generated alarm: DBResourceManager.getData - No active DB connection pools are available.");
 387                         throw new DBLibException("No active DB connection pools are available in RequestDataNoRecovery call.");
 388                 }
 389                 CachedDataSource active = (CachedDataSource) this.dsQueue.peek();
 390                 long time = System.currentTimeMillis();
 391                 try {
 392                         if(!active.isFabric()) {
 393                                 CachedDataSource master = findMaster();
 394                                 if(master != null)
 395                                         active = master;
 396                         }
 397                         return active.getData(statement, arguments);
 398 //              } catch(SQLDataException exc){
 399 //                      throw exc;
 400                 } catch(Throwable exc){
 401                         String message = exc.getMessage();
 402                         if(message == null)
 403                                 message = exc.getClass().getName();
 404                         LOGGER.error("Generated alarm: "+active.getDbConnectionName()+" - "+message);
 405                         if(exc instanceof SQLException)
 406                                 throw (SQLException)exc;
 407                         else {
 408                                 DBLibException excptn = new DBLibException(exc.getMessage());
 409                                 excptn.setStackTrace(exc.getStackTrace());
 410                                 throw excptn;
 411                         }
 412                 } finally {
 413                         if(LOGGER.isDebugEnabled()){
 414                                 time = (System.currentTimeMillis() - time);
 415                                 LOGGER.debug(">> getData : "+ active.getDbConnectionName()+"  "+time+" miliseconds.");
 416                         }
 417                 }
 418         }
 419
 420
 421         /* (non-Javadoc)
 422          * @see org.openecomp.dblib.DataAccessor#writeData(java.lang.String, java.util.ArrayList)
 423          */
 424         /* (non-Javadoc)
 425          * @see org.openecomp.sdnc.sli.resource.dblib.DbLibService#writeData(java.lang.String, java.util.ArrayList, java.lang.String)
 426          */
 427         @Override
 428         public boolean writeData(String statement, ArrayList<String> arguments, String preferredDS) throws SQLException {
 429                 return writeDataNoRecovery(statement, arguments, preferredDS);
 430         }
 431
 432         CachedDataSource findMaster() {
 433                 CachedDataSource master = null;
 434                 CachedDataSource[] dss = this.dsQueue.toArray(new CachedDataSource[0]);
 435                 for(int i=0; i<dss.length; i++) {
 436                         if(!dss[i].isSlave()) {
 437                                 master = dss[i];
 438                                 if(i != 0) {
 439                                         dsQueue.remove(master);
 440                                         dsQueue.add(master);
 441                                 }
 442                                 return master;
 443                         }
 444                 }
 445                 if(master == null) {
 446                         LOGGER.warn("MASTER not found.");
 447                 }
 448                 return master;
 449         }
 450
 451
 452         private boolean writeDataNoRecovery(String statement, ArrayList<String> arguments, String preferredDS) throws SQLException {
 453                 if(dsQueue.isEmpty()){
 454                         LOGGER.error("Generated alarm: DBResourceManager.getData - No active DB connection pools are available.");
 455                         throw new DBLibException("No active DB connection pools are available in RequestDataNoRecovery call.");
 456                 }
 457
 458                 boolean initialRequest = true;
 459                 boolean retryAllowed = true;
 460                 CachedDataSource active = (CachedDataSource) this.dsQueue.peek();
 461                 long time = System.currentTimeMillis();
 462                 while(initialRequest) {
 463                         initialRequest = false;
 464                         try {
 465                                 if(!active.isFabric()) {
 466                                         CachedDataSource master = findMaster();
 467                                         if(master != null) {
 468                                                 active = master;
 469                                         }
 470                                 }
 471
 472                                 return active.writeData(statement, arguments);
 473                         } catch(Throwable exc){
 474                                 String message = exc.getMessage();
 475                                 if(message == null)
 476                                         message = exc.getClass().getName();
 477                                 LOGGER.error("Generated alarm: "+active.getDbConnectionName()+" - "+message);
 478                                 if(exc instanceof SQLException) {
 479                                         SQLException sqlExc = SQLException.class.cast(exc);
 480                                         // handle read-only exception
 481                                         if(sqlExc.getErrorCode() == 1290 && "HY000".equals(sqlExc.getSQLState())) {
 482                                                 LOGGER.warn("retrying due to: " + sqlExc.getMessage());
 483                                                 dsQueue.remove(active);
 484                                                 dsQueue.add(active);
 485                                                 if(retryAllowed){
 486                                                         retryAllowed = false;
 487                                                         initialRequest = true;
 488                                                         continue;
 489                                                 }
 490                                         }
 491                                         throw (SQLException)exc;
 492                                 } else {
 493                                         DBLibException excptn = new DBLibException(exc.getMessage());
 494                                         excptn.setStackTrace(exc.getStackTrace());
 495                                         throw excptn;
 496                                 }
 497                         } finally {
 498                                 if(LOGGER.isDebugEnabled()){
 499                                         time = (System.currentTimeMillis() - time);
 500                                         LOGGER.debug(">> getData : "+ active.getDbConnectionName()+"  "+time+" miliseconds.");
 501                                 }
 502                         }
 503                 }
 504                 return true;
 505         }
 506
 507         private void setDataSource(CachedDataSource dataSource) {
 508                 if(dataSource.testConnection(true)){
 509                         this.dsQueue.add(dataSource);
 510                 } else {
 511                         this.broken.add(dataSource);
 512                 }
 513         }
 514
 515         public Connection getConnection() throws SQLException {
 516                 Throwable lastException = null;
 517                 CachedDataSource active = null;
 518
 519                 if(dsQueue.isEmpty()){
 520                         throw new DBLibException("No active DB connection pools are available in GetConnection call.");
 521                 }
 522
 523                 try {
 524                         active = dsQueue.peek();
 525                         CachedDataSource tmpActive = findMaster();
 526                         if(tmpActive != null) {
 527                                 active = tmpActive;
 528                         }
 529                         return active.getConnection();
 530                 } catch(javax.sql.rowset.spi.SyncFactoryException exc){
 531                         LOGGER.debug("Free memory (bytes): " + Runtime.getRuntime().freeMemory());
 532                         LOGGER.warn("CLASSPATH issue. Allowing retry", exc);
 533                         lastException = exc;
 534                 } catch(Exception exc){
 535                         lastException = exc;
 536                         if(recoveryMode){
 537                                 handleGetConnectionException(active, exc);
 538                         } else {
 539                                 if(exc instanceof SQLException)
 540                                         throw (SQLException)exc;
 541                                 else {
 542                                         DBLibException excptn = new DBLibException(exc.getMessage());
 543                                         excptn.setStackTrace(exc.getStackTrace());
 544                                         throw excptn;
 545                                 }
 546                         }
 547                 } catch (Throwable trwb) {
 548                         DBLibException excptn = new DBLibException(trwb.getMessage());
 549                         excptn.setStackTrace(trwb.getStackTrace());
 550                         throw excptn;
 551                 } finally {
 552                         if(LOGGER.isDebugEnabled()){
 553                                 displayState();
 554                         }
 555                 }
 556
 557                 if(lastException instanceof SQLException){
 558                         throw (SQLException)lastException;
 559                 }
 560                 // repackage the exception
 561                 if(lastException == null) {
 562                         throw new DBLibException("The operation timed out while waiting to acquire a new connection." );
 563                 } else {
 564                         SQLException exception = new DBLibException(lastException.getMessage());
 565                         exception.setStackTrace(lastException.getStackTrace());
 566                         if(lastException.getCause() instanceof SQLException) {
 567 //                              exception.setNextException((SQLException)lastException.getCause());
 568                                 throw (SQLException)lastException.getCause();
 569                         }
 570                         throw exception;
 571                 }
 572         }
 573
 574         public Connection getConnection(String username, String password)
 575         throws SQLException {
 576                 CachedDataSource active = null;
 577
 578                 if(dsQueue.isEmpty()){
 579                         throw new DBLibException("No active DB connection pools are available in GetConnection call.");
 580                 }
 581
 582
 583                 try {
 584                         active = dsQueue.peek();
 585                         CachedDataSource tmpActive = findMaster();
 586                         if(tmpActive != null) {
 587                                 active = tmpActive;
 588                         }
 589                         return active.getConnection(username, password);
 590                 } catch(Throwable exc){
 591                         if(recoveryMode){
 592                                 handleGetConnectionException(active, exc);
 593                         } else {
 594                                 if(exc instanceof SQLException)
 595                                         throw (SQLException)exc;
 596                                 else {
 597                                         DBLibException excptn = new DBLibException(exc.getMessage());
 598                                         excptn.setStackTrace(exc.getStackTrace());
 599                                         throw excptn;
 600                                 }
 601                         }
 602
 603                 }
 604
 605                 throw new DBLibException("No connections available in DBResourceManager in GetConnection call.");
 606         }
 607
 608         private void handleGetConnectionException(CachedDataSource source, Throwable exc) {
 609                 try {
 610                         if(!source.canTakeOffLine())
 611                         {
 612                                 LOGGER.error("Could not switch due to blocking");
 613                                 return;
 614                         }
 615
 616                         boolean removed = dsQueue.remove(source);
 617                         if(!broken.contains(source))
 618                         {
 619                                 if(broken.add(source))
 620                                 {
 621                                         LOGGER.warn("DB Recovery: DataSource <" + source.getDbConnectionName()  + "> put in the recovery mode. Reason : " + exc.getMessage());
 622                                 } else {
 623                                         LOGGER.warn("Error putting DataSource <" +source.getDbConnectionName()+  "> in recovery mode.");
 624                                 }
 625                         } else {
 626                                 LOGGER.info("DB Recovery: DataSource <" + source.getDbConnectionName() + "> already in recovery queue");
 627                         }
 628                         if(removed)
 629                         {
 630                                 if(!dsQueue.isEmpty())
 631                                 {
 632                                         LOGGER.warn("DB DataSource <" + dsQueue.peek().getDbConnectionName()    + "> became active");
 633                                 }
 634                         }
 635                 } catch (Exception e) {
 636                         LOGGER.error("", e);
 637                 }
 638         }
 639
 640         public void cleanUp() {
 641                 for(Iterator<CachedDataSource> it=dsQueue.iterator();it.hasNext();){
 642                         CachedDataSource cds = (CachedDataSource)it.next();
 643                         it.remove();
 644                         cds.cleanUp();
 645                 }
 646
 647                 try {
 648                         this.terminating = true;
 649                         if(broken != null)
 650                         {
 651                                 try {
 652                                         broken.add( new TerminatingCachedDataSource(null));
 653                                 } catch(Exception exc){
 654                                         LOGGER.error("Waiting for Worker to stop", exc);
 655                                 }
 656                         }
 657                         worker.join(terminationTimeOut);
 658                         LOGGER.info("DBResourceManager.RecoveryMgr <"+worker.toString() +"> termination was successful: " + worker.getState());
 659                 } catch(Exception exc){
 660                         LOGGER.error("Waiting for Worker thread to terminate ", exc);
 661                 }
 662         }
 663
 664         public static DBResourceManager create(Properties props) throws Exception {
 665                 DBResourceManager dbmanager = new DBResourceManager(props);
 666                 dbmanager.config(props);
 667                 return dbmanager;
 668         }
 669
 670         public PrintWriter getLogWriter() throws SQLException {
 671                 return ((CachedDataSource)this.dsQueue.peek()).getLogWriter();
 672         }
 673
 674         public int getLoginTimeout() throws SQLException {
 675                 return ((CachedDataSource)this.dsQueue.peek()).getLoginTimeout();
 676         }
 677
 678         public void setLogWriter(PrintWriter out) throws SQLException {
 679                 ((CachedDataSource)this.dsQueue.peek()).setLogWriter(out);
 680         }
 681
 682         public void setLoginTimeout(int seconds) throws SQLException {
 683                 ((CachedDataSource)this.dsQueue.peek()).setLoginTimeout(seconds);
 684         }
 685
 686         public void displayState(){
 687                 if(LOGGER.isDebugEnabled()){
 688                         LOGGER.debug("POOLS : Active = "+dsQueue.size() + ";\t Broken = "+broken.size());
 689                         CachedDataSource current = (CachedDataSource)dsQueue.peek();
 690                         if(current != null) {
 691                                 LOGGER.debug("POOL : Active name = \'"+current.getDbConnectionName()+ "\'");
 692                         }
 693                 }
 694         }
 695
 696         /* (non-Javadoc)
 697          * @see org.openecomp.sdnc.sli.resource.dblib.DbLibService#isActive()
 698          */
 699         @Override
 700         public boolean isActive() {
 701                 return this.dsQueue.size()>0;
 702         }
 703
 704         public String getActiveStatus(){
 705                 return "Connected: " + dsQueue.size()+"\tIn-recovery: "+broken.size();
 706         }
 707
 708         public String getDBStatus(boolean htmlFormat) {
 709                 StringBuilder buffer = new StringBuilder();
 710
 711                 ArrayList<CachedDataSource> list = new ArrayList<CachedDataSource>();
 712                 list.addAll(dsQueue);
 713                 list.addAll(broken);
 714                 if (htmlFormat)
 715                 {
 716                         buffer.append("<tr class=\"headerRow\"><th id=\"header1\">")
 717                                         .append("Name:").append("</th>");
 718                         for (int i = 0; i < list.size(); i++) {
 719                                 buffer.append("<th id=\"header").append(2 + i).append("\">");
 720                                 buffer.append(((CachedDataSource) list.get(i)).getDbConnectionName()).append("</th>");
 721                         }
 722                         buffer.append("</tr>");
 723
 724                         buffer.append("<tr><td>State:</td>");
 725                         for (int i = 0; i < list.size(); i++) {
 726                                 if (broken.contains(list.get(i))) {
 727                                         buffer.append("<td>in recovery</td>");
 728                                 }
 729                                 if (dsQueue.contains(list.get(i))) {
 730                                         if (dsQueue.peek() == list.get(i))
 731                                                 buffer.append("<td>active</td>");
 732                                         else
 733                                                 buffer.append("<td>standby</td>");
 734                                 }
 735                         }
 736                         buffer.append("</tr>");
 737
 738                 } else {
 739                         for (int i = 0; i < list.size(); i++) {
 740                                 buffer.append("Name: ").append(((CachedDataSource) list.get(i)).getDbConnectionName());
 741                                 buffer.append("\tState: ");
 742                                 if (broken.contains(list.get(i))) {
 743                                         buffer.append("in recovery");
 744                                 } else
 745                                 if (dsQueue.contains(list.get(i))) {
 746                                         if (dsQueue.peek() == list.get(i))
 747                                                 buffer.append("active");
 748                                         else
 749                                                 buffer.append("standby");
 750                                 }
 751
 752                                 buffer.append("\n");
 753
 754                         }
 755                 }
 756                 return buffer.toString();
 757         }
 758
 759         public boolean isWrapperFor(Class<?> iface) throws SQLException {
 760                 return false;
 761         }
 762
 763         public <T> T unwrap(Class<T> iface) throws SQLException {
 764                 return null;
 765         }
 766
 767         /**
 768          * @return the monitorDbResponse
 769          */
 770         public final boolean isMonitorDbResponse() {
 771                 return recoveryMode && monitorDbResponse;
 772         }
 773
 774         public void test(){
 775                 CachedDataSource obj = dsQueue.peek();
 776                 Exception ption = new Exception();
 777                 try {
 778                         for(int i=0; i<5; i++)
 779                         {
 780                                 handleGetConnectionException(obj, ption);
 781                         }
 782                 } catch(Throwable exc){
 783                         LOGGER.warn("", exc);
 784                 }
 785         }
 786
 787         public String getPreferredDSName(){
 788                 if(isActive()){
 789                         return getPreferredDataSourceName(dsSelector);
 790                 }
 791                 return "";
 792         }
 793
 794         private String getPreferredDataSourceName(AtomicBoolean flipper) {
 795
 796                 LinkedList<CachedDataSource> snapshot = new LinkedList<CachedDataSource>(dsQueue);
 797                 if(snapshot.size() > 1){
 798                         CachedDataSource first = snapshot.getFirst();
 799                         CachedDataSource last = snapshot.getLast();
 800
 801                         int delta = first.getMonitor().getPorcessedConnectionsCount() - last.getMonitor().getPorcessedConnectionsCount();
 802                         if(delta < 0) {
 803                                 flipper.set(false);
 804                         } else if(delta > 0) {
 805                                 flipper.set(true);
 806                         } else {
 807                                 // check the last value and return !last
 808                                 flipper.getAndSet(!flipper.get());
 809                         }
 810
 811                         if (flipper.get())
 812                                 Collections.reverse(snapshot);
 813                 }
 814                 return snapshot.peek().getDbConnectionName();
 815         }
 816
 817         public java.util.logging.Logger getParentLogger()
 818                         throws SQLFeatureNotSupportedException {
 819                 // TODO Auto-generated method stub
 820                 return null;
 821         }
 822
 823         public String getMasterName() {
 824                 if(isActive()){
 825                         return getMasterDataSourceName(dsSelector);
 826                 }
 827                 return "";
 828         }
 829
 830
 831         private String getMasterDataSourceName(AtomicBoolean flipper) {
 832
 833                 LinkedList<CachedDataSource> snapshot = new LinkedList<CachedDataSource>(dsQueue);
 834                 if(snapshot.size() > 1){
 835                         CachedDataSource first = snapshot.getFirst();
 836                         CachedDataSource last = snapshot.getLast();
 837
 838                         int delta = first.getMonitor().getPorcessedConnectionsCount() - last.getMonitor().getPorcessedConnectionsCount();
 839                         if(delta < 0) {
 840                                 flipper.set(false);
 841                         } else if(delta > 0) {
 842                                 flipper.set(true);
 843                         } else {
 844                                 // check the last value and return !last
 845                                 flipper.getAndSet(!flipper.get());
 846                         }
 847
 848                         if (flipper.get())
 849                                 Collections.reverse(snapshot);
 850                 }
 851                 return snapshot.peek().getDbConnectionName();
 852         }
 853
 854         /*
 855         private void runTest(){
 856                 Thread producer = null;
 857
 858                 producer = new ProducerThread("Prod1");
 859                 producer.setDaemon(true);
 860                 producer.start();
 861
 862                 producer = new ProducerThread("Prod2");
 863                 producer.setDaemon(true);
 864                 producer.start();
 865
 866                 producer = new ProducerThread("Prod3");
 867                 producer.setDaemon(true);
 868                 producer.start();
 869
 870                 producer = new ProducerThread("Prod4");
 871                 producer.setDaemon(true);
 872                 producer.start();
 873
 874                 producer = new ProducerThread("Prod5");
 875                 producer.setDaemon(true);
 876                 producer.start();
 877
 878                 producer = new ProducerThread("Prod6");
 879                 producer.setDaemon(true);
 880                 producer.start();
 881
 882                 producer = new ProducerThread("Prod7");
 883                 producer.setDaemon(true);
 884                 producer.start();
 885
 886                 producer = new ProducerThread("Prod8");
 887                 producer.setDaemon(true);
 888                 producer.start();
 889
 890                 producer = new ProducerThread("Prod9");
 891                 producer.setDaemon(true);
 892                 producer.start();
 893
 894                 producer = new ProducerThread("Pro10");
 895                 producer.setDaemon(true);
 896                 producer.start();
 897
 898         }
 899
 900         private final class ProducerThread extends Thread {
 901                 private ProducerThread(String threadName) {
 902                         super(threadName);
 903                 }
 904
 905                 public void run()
 906                 {
 907                         String name = null;
 908                         for(int i=0; i<(Integer.MAX_VALUE-1); i++)
 909                         {
 910                                         try {
 911                                                 name = getPreferredDataSourceName(dsSelector);
 912                                                 if(name.contains("BACK")){
 913                                                         LOGGER.error(this.getName()+": <======      ");
 914                                                 } else {
 915                                                         LOGGER.error(this.getName()+":       ======>");
 916                                                 }
 917                                                 CachedRowSet rs = null;
 918                                                 rs = getData("select 1 from dual", new ArrayList<String>(), name);
 919                                                 rs.close();
 920                                                 rs = getData("select 1 from dual", new ArrayList<String>(), name);
 921                                                 rs.close();
 922                                                 rs = getData("select 1 from dual", new ArrayList<String>(), name);
 923                                                 rs.close();
 924                                                 rs = getData("select 1 from dual", new ArrayList<String>(), name);
 925                                                 rs.close();
 926                                         } catch (Exception e) {
 927                                                 e.printStackTrace();
 928                                         }
 929
 930                                 try {
 931                                         Thread.sleep(50L);
 932                                 } catch (InterruptedException e) {
 933                                         e.printStackTrace();
 934                                 }
 935                         }
 936                         return;
 937                 }
 938         }
 939 */
 940 }