dblib/provider/src/main/java/org/openecomp/sdnc/sli/resource/dblib/DBResourceManager.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * openecomp
   4  * ================================================================================
   5  * Copyright (C) 2016 - 2017 AT&T
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdnc.sli.resource.dblib;
  22
  23 import java.io.PrintWriter;
  24 import java.sql.Connection;
  25 import java.sql.SQLDataException;
  26 import java.sql.SQLException;
  27 import java.sql.SQLFeatureNotSupportedException;
  28 import java.sql.SQLIntegrityConstraintViolationException;
  29 import java.sql.SQLSyntaxErrorException;
  30 import java.util.ArrayList;
  31 import java.util.Collections;
  32 import java.util.Comparator;
  33 import java.util.HashSet;
  34 import java.util.Iterator;
  35 import java.util.LinkedList;
  36 import java.util.Observable;
  37 import java.util.PriorityQueue;
  38 import java.util.Properties;
  39 import java.util.Queue;
  40 import java.util.Set;
  41 import java.util.concurrent.ConcurrentLinkedQueue;
  42 import java.util.concurrent.atomic.AtomicBoolean;
  43
  44 import javax.sql.DataSource;
  45 import javax.sql.rowset.CachedRowSet;
  46
  47 import org.openecomp.sdnc.sli.resource.dblib.config.DbConfigPool;
  48 import org.openecomp.sdnc.sli.resource.dblib.factory.AbstractDBResourceManagerFactory;
  49 import org.openecomp.sdnc.sli.resource.dblib.factory.AbstractResourceManagerFactory;
  50 import org.openecomp.sdnc.sli.resource.dblib.factory.DBConfigFactory;
  51 import org.openecomp.sdnc.sli.resource.dblib.pm.PollingWorker;
  52 import org.openecomp.sdnc.sli.resource.dblib.pm.SQLExecutionMonitor;
  53 import org.slf4j.Logger;
  54 import org.slf4j.LoggerFactory;
  55
  56
  57 /**
  58  * @version $Revision: 1.15 $
  59  * Change Log
  60  * Author         Date     Comments
  61  * ============== ======== ====================================================
  62  * Rich Tabedzki
  63  */
  64 public class DBResourceManager implements DataSource, DataAccessor, DBResourceObserver, DbLibService {
  65         private static Logger LOGGER = LoggerFactory.getLogger(DBResourceManager.class);
  66
  67         transient boolean terminating = false;
  68         transient protected long retryInterval = 10000L;
  69         transient boolean recoveryMode = true;
  70
  71         protected final AtomicBoolean dsSelector = new  AtomicBoolean();
  72
  73 //      Queue<CachedDataSource> dsQueue = new ConcurrentLinkedQueue<CachedDataSource>();
  74         Queue<CachedDataSource> dsQueue = new PriorityQueue<CachedDataSource>(4, new Comparator<CachedDataSource>(){
  75
  76                 @Override
  77                 public int compare(CachedDataSource left, CachedDataSource right) {
  78                         try {
  79                                 if(!left.isSlave())
  80                                         return -1;
  81                                 if(!right.isSlave())
  82                                         return 1;
  83
  84                         } catch (Throwable e) {
  85                                 LOGGER.warn("", e);
  86                         }
  87                         return 0;
  88                 }
  89
  90         });
  91         protected final Set<CachedDataSource> broken = Collections.synchronizedSet(new HashSet<CachedDataSource>());
  92         protected final Object monitor = new Object();
  93         protected final Properties configProps;
  94         protected final Thread worker;
  95
  96         protected final long terminationTimeOut;
  97         protected final boolean monitorDbResponse;
  98         protected final long monitoringInterval;
  99         protected final long monitoringInitialDelay;
 100         protected final long expectedCompletionTime;
 101         protected final long unprocessedFailoverThreshold;
 102
 103         public DBResourceManager(Properties props){
 104                 this.configProps = props;
 105
 106                 // get retry interval value
 107                 retryInterval = getLongFromProperties(props, "org.openecomp.dblib.connection.retry", 10000L);
 108
 109                 // get recovery mode flag
 110                 recoveryMode = getBooleanFromProperties(props, "org.openecomp.dblib.connection.recovery", true);
 111                 if(!recoveryMode)
 112                 {
 113                         recoveryMode = false;
 114                         LOGGER.info("Recovery Mode disabled");
 115                 }
 116                 // get time out value for thread cleanup
 117                 terminationTimeOut = getLongFromProperties(props, "org.openecomp.dblib.termination.timeout", 300000L);
 118                 // get properties for monitoring
 119                 monitorDbResponse = getBooleanFromProperties(props, "org.openecomp.dblib.connection.monitor", false);
 120                 monitoringInterval = getLongFromProperties(props, "org.openecomp.dblib.connection.monitor.interval", 1000L);
 121                 monitoringInitialDelay = getLongFromProperties(props, "org.openecomp.dblib.connection.monitor.startdelay", 5000L);
 122                 expectedCompletionTime = getLongFromProperties(props, "org.openecomp.dblib.connection.monitor.expectedcompletiontime", 5000L);
 123                 unprocessedFailoverThreshold = getLongFromProperties(props, "org.openecomp.dblib.connection.monitor.unprocessedfailoverthreshold", 3L);
 124
 125                 // initialize performance monitor
 126                 PollingWorker.createInistance(props);
 127
 128                 // initialize recovery thread
 129                 worker = new RecoveryMgr();
 130                 worker.setName("DBResourcemanagerWatchThread");
 131                 worker.setDaemon(true);
 132                 worker.start();
 133         }
 134
 135         private void config(Properties ctx) throws Exception {
 136
 137                 DbConfigPool dbConfig = DBConfigFactory.createConfig(this.configProps);
 138
 139                 try {
 140                         AbstractResourceManagerFactory factory =  AbstractDBResourceManagerFactory.getFactory(dbConfig.getType());
 141                         if(LOGGER.isInfoEnabled()){
 142                                 LOGGER.info("Default DB config is : " + dbConfig.getType());
 143                                 LOGGER.info("Using factory : " + factory.getClass().getName());
 144                         }
 145                         CachedDataSource[] cachedDS = factory.initDBResourceManager(dbConfig, this);
 146                         if(cachedDS == null || cachedDS.length == 0) {
 147                                 LOGGER.error("Initialization of CachedDataSources failed. No instance was created.");
 148                                 throw new Exception("Failed to initialize DB Library. No data source was created.");
 149                         }
 150
 151                         for(int i=0; i<cachedDS.length; i++){
 152                                 if(cachedDS[i] != null && cachedDS[i].isInitialized()){
 153                                         setDataSource(cachedDS[i]);
 154                                         cachedDS[i].setInterval(monitoringInterval);
 155                                         cachedDS[i].setInitialDelay(monitoringInitialDelay);
 156                                         cachedDS[i].setExpectedCompletionTime(expectedCompletionTime);
 157                                         cachedDS[i].setUnprocessedFailoverThreshold(unprocessedFailoverThreshold);
 158                                         cachedDS[i].addObserver(this);
 159                                 }
 160                         }
 161
 162                 } catch(Exception exc){
 163                         throw exc;
 164                 }
 165         }
 166
 167         private long getLongFromProperties(Properties props, String property, long defaultValue)
 168         {
 169                 String value = null;
 170                 long tmpLongValue = defaultValue;
 171                 try {
 172                         value = (String)props.getProperty(property);
 173                         if(value != null)
 174                                 tmpLongValue = Long.parseLong(value);
 175
 176                 } catch(NumberFormatException exc) {
 177                         if(LOGGER.isWarnEnabled()){
 178                                 LOGGER.warn("'"+property+"'=" + value+" is invalid. It should be a numeric value");
 179                         }
 180                 } catch(Exception exc) {
 181                 }
 182                 return tmpLongValue;
 183
 184         }
 185
 186         private boolean getBooleanFromProperties(Properties props, String property, boolean defaultValue)
 187         {
 188                 boolean tmpValue = defaultValue;
 189                 String value = null;
 190
 191                 try {
 192                         value = (String)props.getProperty(property);
 193                         if(value != null)
 194                                 tmpValue = Boolean.parseBoolean(value);
 195
 196                 } catch(NumberFormatException exc) {
 197                         if(LOGGER.isWarnEnabled()){
 198                                 LOGGER.warn("'"+property+"'=" + value+" is invalid. It should be a boolean value");
 199                         }
 200                 } catch(Exception exc) {
 201                 }
 202                 return tmpValue;
 203
 204         }
 205
 206
 207         public void update(Observable observable, Object data) {
 208                 // if observable is active and there is a standby available, switch
 209                 if(observable instanceof SQLExecutionMonitor)
 210                 {
 211                         SQLExecutionMonitor monitor = (SQLExecutionMonitor)observable;
 212                         if(monitor.getParent() instanceof CachedDataSource)
 213                         {
 214                                 CachedDataSource dataSource = (CachedDataSource)monitor.getParent();
 215                                 if(dataSource == dsQueue.peek())
 216                                 {
 217                                         if(recoveryMode && dsQueue.size() > 1){
 218                                                 handleGetConnectionException(dataSource, new Exception(data.toString()));
 219                                         }
 220                                 }
 221                         }
 222                 }
 223         }
 224
 225         public void testForceRecovery()
 226         {
 227                 CachedDataSource active = (CachedDataSource) this.dsQueue.peek();
 228                 handleGetConnectionException(active, new Exception("test"));
 229         }
 230
 231         class RecoveryMgr extends Thread {
 232
 233                 public void run() {
 234                         while(!terminating)
 235                         {
 236                                 try {
 237                                         Thread.sleep(retryInterval);
 238                                 } catch (InterruptedException e1) {     }
 239                                 CachedDataSource brokenSource = null;
 240                                 try {
 241                                         if (!broken.isEmpty()) {
 242                                                 CachedDataSource[] sourceArray = broken.toArray(new CachedDataSource[0]);
 243                                                 for (int i = 0; i < sourceArray.length; i++)
 244                                                 {
 245                                                         brokenSource = sourceArray[i];
 246                                                         if (brokenSource instanceof TerminatingCachedDataSource)
 247                                                                 break;
 248                                                         if (resetConnectionPool(brokenSource)) {
 249                                                                 broken.remove(brokenSource);
 250                                                                 brokenSource.blockImmediateOffLine();
 251                                                                 dsQueue.add(brokenSource);
 252                                                                 LOGGER.info("DataSource <"
 253                                                                                 + brokenSource.getDbConnectionName()
 254                                                                                 + "> recovered.");
 255                                                         }
 256                                                         brokenSource = null;
 257                                                 }
 258                                         }
 259                                 } catch (Exception exc) {
 260                                         LOGGER.warn(exc.getMessage());
 261                                         if(brokenSource != null){
 262                                                 try {
 263                                                         if(!broken.contains(brokenSource))
 264                                                                 broken.add(brokenSource);
 265                                                         brokenSource = null;
 266                                                 } catch (Exception e1) {        }
 267                                         }
 268                                 }
 269                         }
 270                         LOGGER.info("DBResourceManager.RecoveryMgr <"+this.toString() +"> terminated." );
 271                 }
 272
 273                 private boolean resetConnectionPool(CachedDataSource dataSource){
 274                         try {
 275                                 return dataSource.testConnection();
 276                         } catch (Exception exc) {
 277                                 LOGGER.info("DataSource <" + dataSource.getDbConnectionName() + "> resetCache failed with error: "+ exc.getMessage());
 278                                 return false;
 279                         }
 280                 }
 281         }
 282
 283         /* (non-Javadoc)
 284          * @see org.openecomp.dblib.DataAccessor#getData(java.lang.String, java.util.ArrayList)
 285          */
 286         /* (non-Javadoc)
 287          * @see org.openecomp.sdnc.sli.resource.dblib.DbLibService#getData(java.lang.String, java.util.ArrayList, java.lang.String)
 288          */
 289         @Override
 290         public CachedRowSet getData(String statement, ArrayList<String> arguments, String preferredDS) throws SQLException {
 291                 if(recoveryMode)
 292                         return requestDataWithRecovery(statement, arguments, preferredDS);
 293                 else
 294                         return requestDataNoRecovery(statement, arguments, preferredDS);
 295         }
 296
 297         private CachedRowSet requestDataWithRecovery(String statement, ArrayList<String> arguments, String preferredDS) throws SQLException {
 298                 Throwable lastException = null;
 299                 CachedDataSource active = null;
 300
 301                 // test if there are any connection pools available
 302                 LinkedList<CachedDataSource> sources = new LinkedList<CachedDataSource>(this.dsQueue);
 303                 if(sources.isEmpty()){
 304                         LOGGER.error("Generated alarm: DBResourceManager.getData - No active DB connection pools are available.");
 305                         throw new DBLibException("No active DB connection pools are available in RequestDataWithRecovery call.");
 306                 }
 307                 if(preferredDS != null && !sources.peek().getDbConnectionName().equals(preferredDS)) {
 308                         Collections.reverse(sources);
 309                 }
 310
 311
 312                 // loop through available data sources to retrieve data.
 313                 while(!sources.isEmpty())
 314                 {
 315                         active = sources.peek();
 316
 317                         long time = System.currentTimeMillis();
 318                         try {
 319                                 if(!active.isFabric()) {
 320                                         CachedDataSource master = findMaster();
 321                                         if(master != null) {
 322                                                 active = master;
 323                                                 master = null;
 324                                         }
 325                                 }
 326                                 sources.remove(active);
 327                                 return active.getData(statement, arguments);
 328                         } catch(SQLDataException exc){
 329                                 throw exc;
 330                         } catch(SQLSyntaxErrorException exc){
 331                                 throw exc;
 332                         } catch(SQLIntegrityConstraintViolationException exc){
 333                                 throw exc;
 334                         } catch(Throwable exc){
 335                                 lastException = exc;
 336                                 String message = exc.getMessage();
 337                                 if(message == null) {
 338                                         if(exc.getCause() != null) {
 339                                                 message = exc.getCause().getMessage();
 340                                         }
 341                                         if(message == null)
 342                                                 message = exc.getClass().getName();
 343                                 }
 344                                 LOGGER.error("Generated alarm: "+active.getDbConnectionName()+" - "+message);
 345                                 handleGetConnectionException(active, exc);
 346                         } finally {
 347                                 if(LOGGER.isDebugEnabled()){
 348                                         time = (System.currentTimeMillis() - time);
 349                                         LOGGER.debug(">> getData : "+ active.getDbConnectionName()+"  "+time+" miliseconds.");
 350                                 }
 351                         }
 352                 }
 353                 if(lastException instanceof SQLException){
 354                         throw (SQLException)lastException;
 355                 }
 356                 // repackage the exception
 357                 // you are here because either you run out of available data sources
 358                 // or the last exception was not of SQLException type.
 359                 // repackage the exception
 360                 if(lastException == null) {
 361                         throw new DBLibException("The operation timed out while waiting to acquire a new connection." );
 362                 } else {
 363                         SQLException exception = new DBLibException(lastException.getMessage());
 364                         exception.setStackTrace(lastException.getStackTrace());
 365                         if(lastException.getCause() instanceof SQLException) {
 366                                 throw (SQLException)lastException.getCause();
 367                         }
 368                         throw exception;
 369                 }
 370         }
 371
 372         private CachedRowSet requestDataNoRecovery(String statement, ArrayList<String> arguments, String preferredDS) throws SQLException {
 373                 if(dsQueue.isEmpty()){
 374                         LOGGER.error("Generated alarm: DBResourceManager.getData - No active DB connection pools are available.");
 375                         throw new DBLibException("No active DB connection pools are available in RequestDataNoRecovery call.");
 376                 }
 377                 CachedDataSource active = (CachedDataSource) this.dsQueue.peek();
 378                 long time = System.currentTimeMillis();
 379                 try {
 380                         if(!active.isFabric()) {
 381                                 CachedDataSource master = findMaster();
 382                                 if(master != null)
 383                                         active = master;
 384                         }
 385                         return active.getData(statement, arguments);
 386 //              } catch(SQLDataException exc){
 387 //                      throw exc;
 388                 } catch(Throwable exc){
 389                         String message = exc.getMessage();
 390                         if(message == null)
 391                                 message = exc.getClass().getName();
 392                         LOGGER.error("Generated alarm: "+active.getDbConnectionName()+" - "+message);
 393                         if(exc instanceof SQLException)
 394                                 throw (SQLException)exc;
 395                         else {
 396                                 DBLibException excptn = new DBLibException(exc.getMessage());
 397                                 excptn.setStackTrace(exc.getStackTrace());
 398                                 throw excptn;
 399                         }
 400                 } finally {
 401                         if(LOGGER.isDebugEnabled()){
 402                                 time = (System.currentTimeMillis() - time);
 403                                 LOGGER.debug(">> getData : "+ active.getDbConnectionName()+"  "+time+" miliseconds.");
 404                         }
 405                 }
 406         }
 407
 408
 409         /* (non-Javadoc)
 410          * @see org.openecomp.dblib.DataAccessor#writeData(java.lang.String, java.util.ArrayList)
 411          */
 412         /* (non-Javadoc)
 413          * @see org.openecomp.sdnc.sli.resource.dblib.DbLibService#writeData(java.lang.String, java.util.ArrayList, java.lang.String)
 414          */
 415         @Override
 416         public boolean writeData(String statement, ArrayList<String> arguments, String preferredDS) throws SQLException {
 417                 return writeDataNoRecovery(statement, arguments, preferredDS);
 418         }
 419
 420         CachedDataSource findMaster() {
 421                 CachedDataSource master = null;
 422                 CachedDataSource[] dss = this.dsQueue.toArray(new CachedDataSource[0]);
 423                 for(int i=0; i<dss.length; i++) {
 424                         if(!dss[i].isSlave()) {
 425                                 master = dss[i];
 426                                 if(i != 0) {
 427                                         dsQueue.remove(master);
 428                                         dsQueue.add(master);
 429                                 }
 430                                 return master;
 431                         }
 432                 }
 433                 if(master == null) {
 434                         LOGGER.warn("MASTER not found.");
 435                 }
 436                 return master;
 437         }
 438
 439
 440         private boolean writeDataNoRecovery(String statement, ArrayList<String> arguments, String preferredDS) throws SQLException {
 441                 if(dsQueue.isEmpty()){
 442                         LOGGER.error("Generated alarm: DBResourceManager.getData - No active DB connection pools are available.");
 443                         throw new DBLibException("No active DB connection pools are available in RequestDataNoRecovery call.");
 444                 }
 445
 446                 boolean initialRequest = true;
 447                 boolean retryAllowed = true;
 448                 CachedDataSource active = (CachedDataSource) this.dsQueue.peek();
 449                 long time = System.currentTimeMillis();
 450                 while(initialRequest) {
 451                         initialRequest = false;
 452                         try {
 453                                 if(!active.isFabric()) {
 454                                         CachedDataSource master = findMaster();
 455                                         if(master != null) {
 456                                                 active = master;
 457                                         }
 458                                 }
 459
 460                                 return active.writeData(statement, arguments);
 461                         } catch(Throwable exc){
 462                                 String message = exc.getMessage();
 463                                 if(message == null)
 464                                         message = exc.getClass().getName();
 465                                 LOGGER.error("Generated alarm: "+active.getDbConnectionName()+" - "+message);
 466                                 if(exc instanceof SQLException) {
 467                                         SQLException sqlExc = SQLException.class.cast(exc);
 468                                         // handle read-only exception
 469                                         if(sqlExc.getErrorCode() == 1290 && "HY000".equals(sqlExc.getSQLState())) {
 470                                                 LOGGER.warn("retrying due to: " + sqlExc.getMessage());
 471                                                 dsQueue.remove(active);
 472                                                 dsQueue.add(active);
 473                                                 if(retryAllowed){
 474                                                         retryAllowed = false;
 475                                                         initialRequest = true;
 476                                                         continue;
 477                                                 }
 478                                         }
 479                                         throw (SQLException)exc;
 480                                 } else {
 481                                         DBLibException excptn = new DBLibException(exc.getMessage());
 482                                         excptn.setStackTrace(exc.getStackTrace());
 483                                         throw excptn;
 484                                 }
 485                         } finally {
 486                                 if(LOGGER.isDebugEnabled()){
 487                                         time = (System.currentTimeMillis() - time);
 488                                         LOGGER.debug(">> getData : "+ active.getDbConnectionName()+"  "+time+" miliseconds.");
 489                                 }
 490                         }
 491                 }
 492                 return true;
 493         }
 494
 495         private void setDataSource(CachedDataSource dataSource) {
 496                 if(dataSource.testConnection(true)){
 497                         this.dsQueue.add(dataSource);
 498                 } else {
 499                         this.broken.add(dataSource);
 500                 }
 501         }
 502
 503         public Connection getConnection() throws SQLException {
 504                 Throwable lastException = null;
 505                 CachedDataSource active = null;
 506
 507                 if(dsQueue.isEmpty()){
 508                         throw new DBLibException("No active DB connection pools are available in GetConnection call.");
 509                 }
 510
 511                 try {
 512                         active = dsQueue.peek();
 513                         CachedDataSource tmpActive = findMaster();
 514                         if(tmpActive != null) {
 515                                 active = tmpActive;
 516                         }
 517                         return active.getConnection();
 518                 } catch(javax.sql.rowset.spi.SyncFactoryException exc){
 519                         LOGGER.debug("Free memory (bytes): " + Runtime.getRuntime().freeMemory());
 520                         LOGGER.warn("CLASSPATH issue. Allowing retry", exc);
 521                         lastException = exc;
 522                 } catch(Exception exc){
 523                         lastException = exc;
 524                         if(recoveryMode){
 525                                 handleGetConnectionException(active, exc);
 526                         } else {
 527                                 if(exc instanceof SQLException)
 528                                         throw (SQLException)exc;
 529                                 else {
 530                                         DBLibException excptn = new DBLibException(exc.getMessage());
 531                                         excptn.setStackTrace(exc.getStackTrace());
 532                                         throw excptn;
 533                                 }
 534                         }
 535                 } catch (Throwable trwb) {
 536                         DBLibException excptn = new DBLibException(trwb.getMessage());
 537                         excptn.setStackTrace(trwb.getStackTrace());
 538                         throw excptn;
 539                 } finally {
 540                         if(LOGGER.isDebugEnabled()){
 541                                 displayState();
 542                         }
 543                 }
 544
 545                 if(lastException instanceof SQLException){
 546                         throw (SQLException)lastException;
 547                 }
 548                 // repackage the exception
 549                 if(lastException == null) {
 550                         throw new DBLibException("The operation timed out while waiting to acquire a new connection." );
 551                 } else {
 552                         SQLException exception = new DBLibException(lastException.getMessage());
 553                         exception.setStackTrace(lastException.getStackTrace());
 554                         if(lastException.getCause() instanceof SQLException) {
 555 //                              exception.setNextException((SQLException)lastException.getCause());
 556                                 throw (SQLException)lastException.getCause();
 557                         }
 558                         throw exception;
 559                 }
 560         }
 561
 562         public Connection getConnection(String username, String password)
 563         throws SQLException {
 564                 CachedDataSource active = null;
 565
 566                 if(dsQueue.isEmpty()){
 567                         throw new DBLibException("No active DB connection pools are available in GetConnection call.");
 568                 }
 569
 570
 571                 try {
 572                         active = dsQueue.peek();
 573                         CachedDataSource tmpActive = findMaster();
 574                         if(tmpActive != null) {
 575                                 active = tmpActive;
 576                         }
 577                         return active.getConnection(username, password);
 578                 } catch(Throwable exc){
 579                         if(recoveryMode){
 580                                 handleGetConnectionException(active, exc);
 581                         } else {
 582                                 if(exc instanceof SQLException)
 583                                         throw (SQLException)exc;
 584                                 else {
 585                                         DBLibException excptn = new DBLibException(exc.getMessage());
 586                                         excptn.setStackTrace(exc.getStackTrace());
 587                                         throw excptn;
 588                                 }
 589                         }
 590
 591                 }
 592
 593                 throw new DBLibException("No connections available in DBResourceManager in GetConnection call.");
 594         }
 595
 596         private void handleGetConnectionException(CachedDataSource source, Throwable exc) {
 597                 try {
 598                         if(!source.canTakeOffLine())
 599                         {
 600                                 LOGGER.error("Could not switch due to blocking");
 601                                 return;
 602                         }
 603
 604                         boolean removed = dsQueue.remove(source);
 605                         if(!broken.contains(source))
 606                         {
 607                                 if(broken.add(source))
 608                                 {
 609                                         LOGGER.warn("DB Recovery: DataSource <" + source.getDbConnectionName()  + "> put in the recovery mode. Reason : " + exc.getMessage());
 610                                 } else {
 611                                         LOGGER.warn("Error putting DataSource <" +source.getDbConnectionName()+  "> in recovery mode.");
 612                                 }
 613                         } else {
 614                                 LOGGER.info("DB Recovery: DataSource <" + source.getDbConnectionName() + "> already in recovery queue");
 615                         }
 616                         if(removed)
 617                         {
 618                                 if(!dsQueue.isEmpty())
 619                                 {
 620                                         LOGGER.warn("DB DataSource <" + dsQueue.peek().getDbConnectionName()    + "> became active");
 621                                 }
 622                         }
 623                 } catch (Exception e) {
 624                         LOGGER.error("", e);
 625                 }
 626         }
 627
 628         public void cleanUp() {
 629                 for(Iterator<CachedDataSource> it=dsQueue.iterator();it.hasNext();){
 630                         CachedDataSource cds = (CachedDataSource)it.next();
 631                         it.remove();
 632                         cds.cleanUp();
 633                 }
 634
 635                 try {
 636                         this.terminating = true;
 637                         if(broken != null)
 638                         {
 639                                 try {
 640                                         broken.add( new TerminatingCachedDataSource(null));
 641                                 } catch(Exception exc){
 642                                         LOGGER.error("Waiting for Worker to stop", exc);
 643                                 }
 644                         }
 645                         worker.join(terminationTimeOut);
 646                         LOGGER.info("DBResourceManager.RecoveryMgr <"+worker.toString() +"> termination was successful: " + worker.getState());
 647                 } catch(Exception exc){
 648                         LOGGER.error("Waiting for Worker thread to terminate ", exc);
 649                 }
 650         }
 651
 652         public static DBResourceManager create(Properties props) throws Exception {
 653                 DBResourceManager dbmanager = new DBResourceManager(props);
 654                 dbmanager.config(props);
 655                 return dbmanager;
 656         }
 657
 658         public PrintWriter getLogWriter() throws SQLException {
 659                 return ((CachedDataSource)this.dsQueue.peek()).getLogWriter();
 660         }
 661
 662         public int getLoginTimeout() throws SQLException {
 663                 return ((CachedDataSource)this.dsQueue.peek()).getLoginTimeout();
 664         }
 665
 666         public void setLogWriter(PrintWriter out) throws SQLException {
 667                 ((CachedDataSource)this.dsQueue.peek()).setLogWriter(out);
 668         }
 669
 670         public void setLoginTimeout(int seconds) throws SQLException {
 671                 ((CachedDataSource)this.dsQueue.peek()).setLoginTimeout(seconds);
 672         }
 673
 674         public void displayState(){
 675                 if(LOGGER.isDebugEnabled()){
 676                         LOGGER.debug("POOLS : Active = "+dsQueue.size() + ";\t Broken = "+broken.size());
 677                         CachedDataSource current = (CachedDataSource)dsQueue.peek();
 678                         if(current != null) {
 679                                 LOGGER.debug("POOL : Active name = \'"+current.getDbConnectionName()+ "\'");
 680                         }
 681                 }
 682         }
 683
 684         /* (non-Javadoc)
 685          * @see org.openecomp.sdnc.sli.resource.dblib.DbLibService#isActive()
 686          */
 687         @Override
 688         public boolean isActive() {
 689                 return this.dsQueue.size()>0;
 690         }
 691
 692         public String getActiveStatus(){
 693                 return "Connected: " + dsQueue.size()+"\tIn-recovery: "+broken.size();
 694         }
 695
 696         public String getDBStatus(boolean htmlFormat) {
 697                 StringBuilder buffer = new StringBuilder();
 698
 699                 ArrayList<CachedDataSource> list = new ArrayList<CachedDataSource>();
 700                 list.addAll(dsQueue);
 701                 list.addAll(broken);
 702                 if (htmlFormat)
 703                 {
 704                         buffer.append("<tr class=\"headerRow\"><th id=\"header1\">")
 705                                         .append("Name:").append("</th>");
 706                         for (int i = 0; i < list.size(); i++) {
 707                                 buffer.append("<th id=\"header").append(2 + i).append("\">");
 708                                 buffer.append(((CachedDataSource) list.get(i)).getDbConnectionName()).append("</th>");
 709                         }
 710                         buffer.append("</tr>");
 711
 712                         buffer.append("<tr><td>State:</td>");
 713                         for (int i = 0; i < list.size(); i++) {
 714                                 if (broken.contains(list.get(i))) {
 715                                         buffer.append("<td>in recovery</td>");
 716                                 }
 717                                 if (dsQueue.contains(list.get(i))) {
 718                                         if (dsQueue.peek() == list.get(i))
 719                                                 buffer.append("<td>active</td>");
 720                                         else
 721                                                 buffer.append("<td>standby</td>");
 722                                 }
 723                         }
 724                         buffer.append("</tr>");
 725
 726                 } else {
 727                         for (int i = 0; i < list.size(); i++) {
 728                                 buffer.append("Name: ").append(((CachedDataSource) list.get(i)).getDbConnectionName());
 729                                 buffer.append("\tState: ");
 730                                 if (broken.contains(list.get(i))) {
 731                                         buffer.append("in recovery");
 732                                 } else
 733                                 if (dsQueue.contains(list.get(i))) {
 734                                         if (dsQueue.peek() == list.get(i))
 735                                                 buffer.append("active");
 736                                         else
 737                                                 buffer.append("standby");
 738                                 }
 739
 740                                 buffer.append("\n");
 741
 742                         }
 743                 }
 744                 return buffer.toString();
 745         }
 746
 747         public boolean isWrapperFor(Class<?> iface) throws SQLException {
 748                 return false;
 749         }
 750
 751         public <T> T unwrap(Class<T> iface) throws SQLException {
 752                 return null;
 753         }
 754
 755         /**
 756          * @return the monitorDbResponse
 757          */
 758         public final boolean isMonitorDbResponse() {
 759                 return recoveryMode && monitorDbResponse;
 760         }
 761
 762         public void test(){
 763                 CachedDataSource obj = dsQueue.peek();
 764                 Exception ption = new Exception();
 765                 try {
 766                         for(int i=0; i<5; i++)
 767                         {
 768                                 handleGetConnectionException(obj, ption);
 769                         }
 770                 } catch(Throwable exc){
 771                         LOGGER.warn("", exc);
 772                 }
 773         }
 774
 775         public String getPreferredDSName(){
 776                 if(isActive()){
 777                         return getPreferredDataSourceName(dsSelector);
 778                 }
 779                 return "";
 780         }
 781
 782         private String getPreferredDataSourceName(AtomicBoolean flipper) {
 783
 784                 LinkedList<CachedDataSource> snapshot = new LinkedList<CachedDataSource>(dsQueue);
 785                 if(snapshot.size() > 1){
 786                         CachedDataSource first = snapshot.getFirst();
 787                         CachedDataSource last = snapshot.getLast();
 788
 789                         int delta = first.getMonitor().getPorcessedConnectionsCount() - last.getMonitor().getPorcessedConnectionsCount();
 790                         if(delta < 0) {
 791                                 flipper.set(false);
 792                         } else if(delta > 0) {
 793                                 flipper.set(true);
 794                         } else {
 795                                 // check the last value and return !last
 796                                 flipper.getAndSet(!flipper.get());
 797                         }
 798
 799                         if (flipper.get())
 800                                 Collections.reverse(snapshot);
 801                 }
 802                 return snapshot.peek().getDbConnectionName();
 803         }
 804
 805         public java.util.logging.Logger getParentLogger()
 806                         throws SQLFeatureNotSupportedException {
 807                 // TODO Auto-generated method stub
 808                 return null;
 809         }
 810
 811         public String getMasterName() {
 812                 if(isActive()){
 813                         return getMasterDataSourceName(dsSelector);
 814                 }
 815                 return "";
 816         }
 817
 818
 819         private String getMasterDataSourceName(AtomicBoolean flipper) {
 820
 821                 LinkedList<CachedDataSource> snapshot = new LinkedList<CachedDataSource>(dsQueue);
 822                 if(snapshot.size() > 1){
 823                         CachedDataSource first = snapshot.getFirst();
 824                         CachedDataSource last = snapshot.getLast();
 825
 826                         int delta = first.getMonitor().getPorcessedConnectionsCount() - last.getMonitor().getPorcessedConnectionsCount();
 827                         if(delta < 0) {
 828                                 flipper.set(false);
 829                         } else if(delta > 0) {
 830                                 flipper.set(true);
 831                         } else {
 832                                 // check the last value and return !last
 833                                 flipper.getAndSet(!flipper.get());
 834                         }
 835
 836                         if (flipper.get())
 837                                 Collections.reverse(snapshot);
 838                 }
 839                 return snapshot.peek().getDbConnectionName();
 840         }
 841
 842         /*
 843         private void runTest(){
 844                 Thread producer = null;
 845
 846                 producer = new ProducerThread("Prod1");
 847                 producer.setDaemon(true);
 848                 producer.start();
 849
 850                 producer = new ProducerThread("Prod2");
 851                 producer.setDaemon(true);
 852                 producer.start();
 853
 854                 producer = new ProducerThread("Prod3");
 855                 producer.setDaemon(true);
 856                 producer.start();
 857
 858                 producer = new ProducerThread("Prod4");
 859                 producer.setDaemon(true);
 860                 producer.start();
 861
 862                 producer = new ProducerThread("Prod5");
 863                 producer.setDaemon(true);
 864                 producer.start();
 865
 866                 producer = new ProducerThread("Prod6");
 867                 producer.setDaemon(true);
 868                 producer.start();
 869
 870                 producer = new ProducerThread("Prod7");
 871                 producer.setDaemon(true);
 872                 producer.start();
 873
 874                 producer = new ProducerThread("Prod8");
 875                 producer.setDaemon(true);
 876                 producer.start();
 877
 878                 producer = new ProducerThread("Prod9");
 879                 producer.setDaemon(true);
 880                 producer.start();
 881
 882                 producer = new ProducerThread("Pro10");
 883                 producer.setDaemon(true);
 884                 producer.start();
 885
 886         }
 887
 888         private final class ProducerThread extends Thread {
 889                 private ProducerThread(String threadName) {
 890                         super(threadName);
 891                 }
 892
 893                 public void run()
 894                 {
 895                         String name = null;
 896                         for(int i=0; i<(Integer.MAX_VALUE-1); i++)
 897                         {
 898                                         try {
 899                                                 name = getPreferredDataSourceName(dsSelector);
 900                                                 if(name.contains("BACK")){
 901                                                         LOGGER.error(this.getName()+": <======      ");
 902                                                 } else {
 903                                                         LOGGER.error(this.getName()+":       ======>");
 904                                                 }
 905                                                 CachedRowSet rs = null;
 906                                                 rs = getData("select 1 from dual", new ArrayList<String>(), name);
 907                                                 rs.close();
 908                                                 rs = getData("select 1 from dual", new ArrayList<String>(), name);
 909                                                 rs.close();
 910                                                 rs = getData("select 1 from dual", new ArrayList<String>(), name);
 911                                                 rs.close();
 912                                                 rs = getData("select 1 from dual", new ArrayList<String>(), name);
 913                                                 rs.close();
 914                                         } catch (Exception e) {
 915                                                 e.printStackTrace();
 916                                         }
 917
 918                                 try {
 919                                         Thread.sleep(50L);
 920                                 } catch (InterruptedException e) {
 921                                         e.printStackTrace();
 922                                 }
 923                         }
 924                         return;
 925                 }
 926         }
 927 */
 928 }