gerrit.onap Code Review - policy/drools-pdp.git/blob

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * feature-active-standby-management
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.policy.drools.activestandby;
  22
  23 import java.util.ArrayList;
  24 import java.util.Collection;
  25 import java.util.Date;
  26 import java.util.Timer;
  27 import java.util.TimerTask;
  28
  29 import org.onap.policy.common.im.StateManagement;
  30 import org.slf4j.Logger;
  31 import org.slf4j.LoggerFactory;
  32 import org.onap.policy.drools.statemanagement.StateManagementFeatureAPI;
  33
  34 public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
  35         // get an instance of logger
  36         private final static Logger  logger = LoggerFactory.getLogger(DroolsPdpsElectionHandler.class);
  37         private DroolsPdpsConnector pdpsConnector;
  38         private Object pdpsConnectorLock = new Object();
  39         private Object checkUpdateWorkerLock = new Object();
  40         private Object checkWaitTimerLock = new Object();
  41         private Object designationWaiterLock = new Object();
  42
  43         /*
  44          * Must be static, so it can be referenced by JpaDroolsPdpsConnector,
  45          * without requiring a reference to the election handler instantiation.
  46          */
  47         private static DroolsPdp myPdp;
  48
  49         private DesignationWaiter designationWaiter;
  50         private Timer updateWorker;
  51         private Timer waitTimer;
  52         private Date updateWorkerLastRunDate;
  53         private Date waitTimerLastRunDate;
  54         private int pdpCheckInterval;
  55         private int pdpUpdateInterval;
  56         private volatile boolean isDesignated;
  57
  58         private String pdpdNowActive;
  59         private String pdpdLastActive;
  60
  61         private StateManagementFeatureAPI stateManagementFeature;
  62
  63         public DroolsPdpsElectionHandler(DroolsPdpsConnector pdps, DroolsPdp myPdp){
  64                 pdpdNowActive = null;
  65                 pdpdLastActive = null;
  66                 this.pdpsConnector = pdps;
  67                 DroolsPdpsElectionHandler.myPdp = myPdp;
  68                 this.isDesignated = false;
  69                 pdpCheckInterval = 3000;
  70                 try{
  71                         pdpCheckInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(ActiveStandbyProperties.PDP_CHECK_INVERVAL));
  72                 }catch(Exception e){
  73                         logger.error
  74                         ("Could not get pdpCheckInterval property. Using default", e);
  75                 }
  76                 pdpUpdateInterval = 2000;
  77                 try{
  78                         pdpUpdateInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(ActiveStandbyProperties.PDP_UPDATE_INTERVAL));
  79                 }catch(Exception e){
  80                         logger.error
  81                         ("Could not get pdpUpdateInterval property. Using default", e);
  82                 }
  83
  84                 Date now = new Date();
  85
  86                 // Retrieve the ms since the epoch
  87                 long nowMs = now.getTime();
  88
  89                 // Create the timer which will update the updateDate in DroolsPdpEntity table.
  90                 // This is the heartbeat
  91                 updateWorker = new Timer();
  92
  93                 // Schedule the heartbeat to start in 100 ms and run at pdpCheckInterval ms thereafter
  94                 // NOTE: The first run of the TimerUpdateClass results in myPdp being added to the
  95                 // drools droolsPdpEntity table.
  96                 updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
  97                 updateWorkerLastRunDate = new Date(nowMs + 100);
  98
  99                 // Create the timer which will run the election algorithm
 100                 waitTimer = new Timer();
 101
 102                 // Schedule it to start in startMs ms (so it will run after the updateWorker and run at pdpUpdateInterval ms thereafter
 103                 long startMs = getDWaiterStartMs();
 104                 designationWaiter = new DesignationWaiter();
 105                 waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
 106                 waitTimerLastRunDate = new Date(nowMs + startMs);
 107
 108                 //Get the StateManagementFeature instance
 109
 110                 for (StateManagementFeatureAPI feature : StateManagementFeatureAPI.impl.getList())
 111                 {
 112                         if (feature.getResourceName().equals(myPdp.getPdpId()))
 113                         {
 114                                 if(logger.isDebugEnabled()){
 115                                         logger.debug("DroolsPdpsElectionHandler: Found StateManagementFeature"
 116                                                 + " with resourceName: {}", myPdp.getPdpId());
 117                                 }
 118                                 stateManagementFeature = feature;
 119                                 break;
 120                         }
 121                 }
 122                 if(stateManagementFeature == null){
 123                         logger.error("DroolsPdpsElectionHandler failed to initialize.  "
 124                                         + "Unable to get instance of StateManagementFeatureAPI "
 125                                         + "with resourceID: {}", myPdp.getPdpId());
 126                 }
 127         }
 128
 129         /*
 130          * When the JpaDroolsPdpsConnector.standDown() method is invoked, it needs
 131          * access to myPdp, so it can keep its designation status in sync with the
 132          * DB.
 133          */
 134         public static void setMyPdpDesignated(boolean designated) {
 135                 if(logger.isDebugEnabled()){
 136                         logger.debug
 137                         ("setMyPdpDesignated: designated= {}", designated);
 138                 }
 139                 myPdp.setDesignated(designated);
 140         }
 141
 142         private class DesignationWaiter extends TimerTask {
 143                 // get an instance of logger
 144                 private final Logger  logger = LoggerFactory.getLogger(DesignationWaiter.class);
 145
 146                 @Override
 147                 public void run() {
 148                         try{
 149                                 if(logger.isDebugEnabled()){
 150                                         logger.debug
 151                                         ("DesignatedWaiter.run: Entering");
 152                                 }
 153
 154                                 // just here initially so code still works
 155                                 if (pdpsConnector == null) {
 156                                         waitTimerLastRunDate = new Date();
 157                                         if(logger.isDebugEnabled()){
 158                                                 logger.debug("DesignatedWaiter.run (pdpsConnector==null) waitTimerLastRunDate = {}", waitTimerLastRunDate);
 159                                         }
 160
 161                                         return;
 162                                 }
 163
 164                                 synchronized (designationWaiterLock) {
 165
 166                                         if(logger.isDebugEnabled()){
 167                                                 logger.debug
 168                                                 ("DesignatedWaiter.run: Entering synchronized block");
 169                                         }
 170
 171                                         checkUpdateWorkerTimer();
 172
 173                                         //It is possible that multiple PDPs are designated lead.  So, we will make a list of all designated
 174                                         //PDPs and then decide which one really should be designated at the end.
 175                                         ArrayList<DroolsPdp> listOfDesignated = new ArrayList<DroolsPdp>();
 176
 177                                         Collection<DroolsPdp> pdps = pdpsConnector.getDroolsPdps();
 178                                         DroolsPdp designatedPdp = null;
 179
 180                                         if(logger.isDebugEnabled()){
 181                                                 logger.debug
 182                                                 ("DesignatedWaiter.run: pdps.size= {}", pdps.size());
 183                                         }
 184
 185                                         //This is only true if all designated PDPs have failed
 186                                         boolean designatedPdpHasFailed = pdpsConnector.hasDesignatedPdpFailed(pdps);
 187                                         if(logger.isDebugEnabled()){
 188                                                 logger.debug
 189                                                 ("DesignatedWaiter.run: designatedPdpHasFailed= {}", designatedPdpHasFailed);
 190                                         }
 191                                         for (DroolsPdp pdp : pdps) {
 192                                                 if(logger.isDebugEnabled()){
 193                                                         logger.debug
 194                                                         ("DesignatedWaiter.run: evaluating pdp ID: {}", pdp.getPdpId());
 195                                                 }
 196
 197                                                 /*
 198                                                  * Note: side effect of isPdpCurrent is that any stale but
 199                                                  * designated PDPs will be marked as un-designated.
 200                                                  */
 201                                                 boolean isCurrent = pdpsConnector.isPdpCurrent(pdp);
 202
 203                                                 /*
 204                                                  * We can't use stateManagement.getStandbyStatus() here, because
 205                                                  * we need the standbyStatus, not for this PDP, but for the PDP
 206                                                  * being processed by this loop iteration.
 207                                                  */
 208                                                 String standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 209                                                 if(standbyStatus==null){
 210                                                         // Treat this case as a cold standby -- if we
 211                                                         // abort here, no sessions will be created in a
 212                                                         // single-node test environment.
 213                                                         standbyStatus = StateManagement.COLD_STANDBY;
 214                                                 }
 215                                                 if(logger.isDebugEnabled()){
 216                                                         logger.debug
 217                                                         ("DesignatedWaiter.run: PDP= {},  isCurrent= {}", pdp.getPdpId(), isCurrent);
 218                                                 }
 219
 220                                                 /*
 221                                                  * There are 4 combinations of isDesignated and isCurrent.  We will examine each one in-turn
 222                                                  * and evaluate the each pdp in the list of pdps against each combination.
 223                                                  *
 224                                                  * This is the first combination of isDesignated and isCurrent
 225                                                  */
 226                                                 if (pdp.isDesignated()  &&  isCurrent) {
 227                                                         //It is current, but it could have a standbystatus=coldstandby / hotstandby
 228                                                         //If so, we need to stand it down and demote it
 229                                                         if(!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)){
 230                                                                 if(pdp.getPdpId().equals(myPdp.getPdpId())){
 231                                                                         if(logger.isDebugEnabled()){
 232                                                                                 logger.debug
 233                                                                                 ("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 234                                                                                         + "butstandbystatus is not providingservice. "
 235                                                                                         + " Executing stateManagement.demote()" + "\n\n", myPdp.getPdpId());
 236                                                                         }
 237                                                                         // So, we must demote it
 238                                                                         try {
 239                                                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 240                                                                                 //This will change isDesignated and it can enter another if(combination) below
 241                                                                                 pdpsConnector.standDownPdp(pdp.getPdpId());
 242                                                                                 myPdp.setDesignated(false);
 243                                                                                 isDesignated = false;
 244                                                                                 if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 245                                                                                                 standbyStatus.equals(StateManagement.COLD_STANDBY))){
 246                                                                                         /*
 247                                                                                          * Only demote it if it appears it has not already been demoted. Don't worry
 248                                                                                          * about synching with the topic endpoint states.  That is done by the
 249                                                                                          * refreshStateAudit
 250                                                                                          */
 251                                                                                         stateManagementFeature.demote();
 252                                                                                 }
 253                                                                                 //update the standbystatus to check in a later combination of isDesignated and isCurrent
 254                                                                                 standbyStatus=stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 255                                                                         } catch (Exception e) {
 256                                                                                 logger.error
 257                                                                                 ("DesignatedWaiter.run: myPdp: {} "
 258                                                                                                 + "Caught Exception attempting to demote myPdp,"
 259                                                                                                 + "message= {}", myPdp.getPdpId(), e);
 260                                                                         }
 261                                                                 }else{
 262                                                                         // Don't demote a remote PDP that is current.  It should catch itself
 263                                                                         if(logger.isDebugEnabled()){
 264                                                                                 logger.debug
 265                                                                                 ("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 266                                                                                         + "but standbystatus is not providingservice. "
 267                                                                                         + " Cannot execute stateManagement.demote() since it it is not myPdp\n\n", myPdp.getPdpId());
 268                                                                         }
 269                                                                 }
 270
 271                                                         }else{
 272                                                                 // If we get here, it is ok to be on the list
 273                                                                 if(logger.isDebugEnabled()){
 274                                                                         logger.debug
 275                                                                         ("DesignatedWaiter.run: PDP= {} is designated, current and {} Noting PDP as "
 276                                                                                 + "designated, standbyStatus= {}", pdp.getPdpId(), standbyStatus, standbyStatus);
 277                                                                 }
 278                                                                 listOfDesignated.add(pdp);
 279                                                         }
 280
 281
 282                                                 }
 283
 284
 285                                                 /*
 286                                                  * The second combination of isDesignated and isCurrent
 287                                                  *
 288                                                  * PDP is designated but not current; it has failed.   So we stand it down (it doesn't matter what
 289                                                  * its standbyStatus is). None of these go on the list.
 290                                                  */
 291                                                 if (pdp.isDesignated()  &&  !isCurrent) {
 292                                                         if(logger.isDebugEnabled()){
 293                                                                 logger.debug
 294                                                                 ("INFO: DesignatedWaiter.run: PDP= {} is currently designated but is not current; "
 295                                                                         + "it has failed.  Standing down.  standbyStatus= {}", pdp.getPdpId(), standbyStatus);
 296                                                         }
 297                                                         /*
 298                                                          * Changes designated to 0 but it is still potentially providing service
 299                                                          * Will affect isDesignated, so, it can enter an if(combination) below
 300                                                          */
 301                                                         pdpsConnector.standDownPdp(pdp.getPdpId());
 302
 303                                                         //need to change standbystatus to coldstandby
 304                                                         if (pdp.getPdpId().equals(myPdp.getPdpId())){
 305                                                                 if(logger.isDebugEnabled()){
 306                                                                         logger.debug
 307                                                                         ("\n\nDesignatedWaiter.run: myPdp {} is not Current. "
 308                                                                                 + " Executing stateManagement.disableFailed()\n\n", myPdp.getPdpId());
 309                                                                 }
 310                                                                 // We found that myPdp is designated but not current
 311                                                                 // So, we must cause it to disableFail
 312                                                                 try {
 313                                                                         myPdp.setDesignated(false);
 314                                                                         pdpsConnector.setDesignated(myPdp, false);
 315                                                                         isDesignated = false;
 316                                                                         stateManagementFeature.disableFailed();
 317                                                                 } catch (Exception e) {
 318                                                                         logger.error
 319                                                                         ("DesignatedWaiter.run: myPdp: {} Caught Exception "
 320                                                                                         + "attempting to disableFail myPdp {}, message= {}",
 321                                                                                         myPdp.getPdpId(), myPdp.getPdpId(), e);
 322                                                                 }
 323                                                         } else { //it is a remote PDP that is failed
 324                                                                 if(logger.isDebugEnabled()){
 325                                                                         logger.debug
 326                                                                         ("\n\nDesignatedWaiter.run: PDP {} is not Current. "
 327                                                                                 + " Executing stateManagement.disableFailed(otherResourceName)\n\n", pdp.getPdpId() );
 328                                                                 }
 329                                                                 // We found a PDP is designated but not current
 330                                                                 // We already called standdown(pdp) which will change designated to false
 331                                                                 // Now we need to disableFail it to get its states in synch.  The standbyStatus
 332                                                                 // should equal coldstandby
 333                                                                 try {
 334                                                                         stateManagementFeature.disableFailed(pdp.getPdpId());
 335                                                                 } catch (Exception e) {
 336                                                                         logger.error
 337                                                                         ("DesignatedWaiter.run: for PDP {}  Caught Exception attempting to "
 338                                                                                         + "disableFail({}), message= {}",
 339                                                                                         pdp.getPdpId(), pdp.getPdpId(), e);
 340                                                                 }
 341
 342                                                         }
 343                                                         continue; //we are not going to do anything else with this pdp
 344                                                 }
 345
 346                                                 /*
 347                                                  * The third combination of isDesignated and isCurrent
 348                                                  * /*
 349                                                  * If a PDP is not currently designated but is providing service (erroneous, but recoverable) or hot standby
 350                                                  * we can add it to the list of possible designated if all the designated have failed
 351                                                  */
 352                                                 if (!pdp.isDesignated() && isCurrent){
 353                                                         if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 354                                                                         standbyStatus.equals(StateManagement.COLD_STANDBY))){
 355                                                                 if(logger.isDebugEnabled()){
 356                                                                         logger.debug("\n\nDesignatedWaiter.run: PDP {}"
 357                                                                                 + " is NOT designated but IS current and"
 358                                                                                 + " has a standbystatus= {}", pdp.getPdpId(), standbyStatus);
 359                                                                 }
 360                                                                 // Since it is current, we assume it can adjust its own state.
 361                                                                 // We will demote if it is myPdp
 362                                                                 if(pdp.getPdpId().equals(myPdp.getPdpId())){
 363                                                                         //demote it
 364                                                                         if(logger.isDebugEnabled()){
 365                                                                                 logger.debug("DesignatedWaiter.run: PDP {} going to "
 366                                                                                         + "setDesignated = false and calling stateManagement.demote",  pdp.getPdpId());
 367                                                                         }
 368                                                                         try {
 369                                                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 370                                                                                 pdpsConnector.setDesignated(myPdp, false);
 371                                                                                 myPdp.setDesignated(false);
 372                                                                                 isDesignated = false;
 373                                                                                 //This is definitely not a redundant call.  It is attempting to correct a problem
 374                                                                                 stateManagementFeature.demote();
 375                                                                                 //recheck the standbystatus
 376                                                                                 standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 377                                                                         } catch (Exception e) {
 378                                                                                 logger.error
 379                                                                                 ("DesignatedWaiter.run: myPdp: {} Caught Exception "
 380                                                                                                 + "attempting to demote myPdp {}, message = {}",  myPdp.getPdpId(),
 381                                                                                                 myPdp.getPdpId(), e);
 382                                                                         }
 383
 384                                                                 }
 385                                                         }
 386                                                         if(standbyStatus.equals(StateManagement.HOT_STANDBY) && designatedPdpHasFailed){
 387                                                                 //add it to the list
 388                                                                 if(logger.isDebugEnabled()){
 389                                                                         logger.debug
 390                                                                         ("INFO: DesignatedWaiter.run: PDP= {}"
 391                                                                                 + " is not designated but is {} and designated PDP "
 392                                                                                 + "has failed.  standbyStatus= {}", pdp.getPdpId(),
 393                                                                                 standbyStatus, standbyStatus);
 394                                                                 }
 395                                                                 listOfDesignated.add(pdp);
 396                                                         }
 397                                                         continue; //done with this one
 398                                                 }
 399
 400                                                 /*
 401                                                  * The fourth combination of isDesignated and isCurrent
 402                                                  *
 403                                                  * We are not going to put any of these on the list since it appears they have failed.
 404
 405                                                  *
 406                                                  */
 407                                                 if(!pdp.isDesignated() && !isCurrent) {
 408                                                         if(logger.isDebugEnabled()){
 409                                                                 logger.debug
 410                                                                 ("INFO: DesignatedWaiter.run: PDP= {} "
 411                                                                         + "designated= {}, current= {}, "
 412                                                                         + "designatedPdpHasFailed= {}, "
 413                                                                         + "standbyStatus= {}",pdp.getPdpId(),
 414                                                                         pdp.isDesignated(), isCurrent, designatedPdpHasFailed, standbyStatus);
 415                                                         }
 416                                                         if(!standbyStatus.equals(StateManagement.COLD_STANDBY)){
 417                                                                 //stand it down
 418                                                                 //disableFail it
 419                                                                 pdpsConnector.standDownPdp(pdp.getPdpId());
 420                                                                 if(pdp.getPdpId().equals(myPdp.getPdpId())){
 421                                                                         /*
 422                                                                          * I don't actually know how this condition could happen, but if it did, we would want
 423                                                                          * to declare it failed.
 424                                                                          */
 425                                                                         if(logger.isDebugEnabled()){
 426                                                                                 logger.debug
 427                                                                                 ("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 428                                                                                         + " Executing stateManagement.disableFailed()\n\n", myPdp.getPdpId());
 429                                                                         }
 430                                                                         // So, we must disableFail it
 431                                                                         try {
 432                                                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 433                                                                                 pdpsConnector.setDesignated(myPdp, false);
 434                                                                                 myPdp.setDesignated(false);
 435                                                                                 isDesignated = false;
 436                                                                                 stateManagementFeature.disableFailed();
 437                                                                         } catch (Exception e) {
 438                                                                                 logger.error
 439                                                                                 ("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 440                                                                                                 + "disableFail myPdp {}, message= {}",
 441                                                                                                  myPdp.getPdpId(), myPdp.getPdpId(), e);
 442                                                                         }
 443                                                                 }else{//it is remote
 444                                                                         if(logger.isDebugEnabled()){
 445                                                                                 logger.debug
 446                                                                                 ("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 447                                                                                         + " Executing stateManagement.disableFailed({})\n\n",
 448                                                                                         myPdp.getPdpId(), pdp.getPdpId());
 449                                                                         }
 450                                                                         // We already called standdown(pdp) which will change designated to false
 451                                                                         // Now we need to disableFail it to get its states in sync.  StandbyStatus = coldstandby
 452                                                                         try {
 453                                                                                 stateManagementFeature.disableFailed(pdp.getPdpId());
 454                                                                         } catch (Exception e) {
 455                                                                                 logger.error
 456                                                                                 ("DesignatedWaiter.run: for PDP {}"
 457                                                                                                 + " Caught Exception attempting to disableFail({})"
 458                                                                                                 + ", message=", pdp.getPdpId(), pdp.getPdpId(), e);
 459                                                                         }
 460                                                                 }
 461                                                         }
 462                                                 }
 463
 464
 465                                         } // end pdps loop
 466
 467                                         /*
 468                                          * We have checked the four combinations of isDesignated and isCurrent.  Where appropriate,
 469                                          * we added the PDPs to the potential list of designated pdps
 470                                          *
 471                                          * We need to give priority to pdps on the same site that is currently being used
 472                                          * First, however, we must sanitize the list of designated to make sure their are
 473                                          * only designated members or non-designated members.  There should not be both in
 474                                          * the list. Because there are real time delays, it is possible that both types could
 475                                          * be on the list.
 476                                          */
 477
 478                                         listOfDesignated = santizeDesignatedList(listOfDesignated);
 479
 480                                         /*
 481                                          * We need to figure out the last pdp that was the primary so we can get the last site
 482                                          * name and the last session numbers.  We need to create a "dummy" droolspdp since
 483                                          * it will be used in later comparisons and cannot be null.
 484                                          */
 485
 486                                         DroolsPdp mostRecentPrimary = computeMostRecentPrimary(pdps, listOfDesignated);
 487
 488                                         if(mostRecentPrimary != null){
 489                                                 pdpdLastActive = mostRecentPrimary.getPdpId();
 490                                         }
 491
 492
 493                                         /*
 494                                          * It is possible to get here with more than one pdp designated and providingservice. This normally
 495                                          * occurs when there is a race condition with multiple nodes coming up at the same time. If that is
 496                                          * the case we must determine which one is the one that should be designated and which one should
 497                                          * be demoted.
 498                                          *
 499                                          * It is possible to have 0, 1, 2 or more but not all, or all designated.
 500                                          *   If we have one designated and current, we chose it and are done
 501                                          *   If we have 2 or more, but not all, we must determine which one is in the same site as
 502                                          *   the previously designated pdp.
 503                                          */
 504
 505                                         designatedPdp = computeDesignatedPdp(listOfDesignated, mostRecentPrimary);
 506                                         if(designatedPdp != null){
 507                                                 pdpdNowActive = designatedPdp.getPdpId();
 508                                         }
 509
 510                                         if (designatedPdp == null) {
 511                                                 logger.warn
 512                                                 ("WARNING: DesignatedWaiter.run: No viable PDP found to be Designated. designatedPdp still null.");
 513                                                 // Just to be sure the parameters are correctly set
 514                                                 myPdp.setDesignated(false);
 515                                                 pdpsConnector.setDesignated(myPdp,false);
 516                                                 isDesignated = false;
 517
 518                                                 waitTimerLastRunDate = new Date();
 519                                                 if(logger.isDebugEnabled()){
 520                                                         logger.debug("DesignatedWaiter.run (designatedPdp == null) waitTimerLastRunDate = {}", waitTimerLastRunDate);
 521                                                 }
 522
 523                                                 return;
 524
 525                                         } else if (designatedPdp.getPdpId().equals(myPdp.getPdpId())) {
 526                                                 if(logger.isDebugEnabled()){
 527                                                         logger.debug
 528                                                         ("DesignatedWaiter.run: designatedPdp is PDP={}", myPdp.getPdpId());
 529                                                 }
 530                                                 /*
 531                                                  * update function expects myPdp.isDesignated to be true.
 532                                                  */
 533                                                 try {
 534                                                         //Keep the order like this.  StateManagement is last since it triggers controller init
 535                                                         myPdp.setDesignated(true);
 536                                                         myPdp.setDesignatedDate(new Date());
 537                                                         pdpsConnector.setDesignated(myPdp, true);
 538                                                         isDesignated = true;
 539                                                         String standbyStatus = stateManagementFeature.getStandbyStatus();
 540                                                         if(!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)){
 541                                                                 /*
 542                                                                  * Only call promote if it is not already in the right state.  Don't worry about
 543                                                                  * synching the lower level topic endpoint states.  That is done by the
 544                                                                  * refreshStateAudit.
 545                                                                  * Note that we need to fetch the session list from 'mostRecentPrimary'
 546                                                                  * at this point -- soon, 'mostRecentPrimary' will be set to this host.
 547                                                                  */
 548                                                                 //this.sessions = mostRecentPrimary.getSessions();
 549                                                                 stateManagementFeature.promote();
 550                                                         }
 551                                                 } catch (Exception e) {
 552                                                         logger.error
 553                                                         ("ERROR: DesignatedWaiter.run: Caught Exception attempting to promote PDP={}"
 554                                                                         + ", message=", myPdp.getPdpId(), e);
 555                                                         myPdp.setDesignated(false);
 556                                                         pdpsConnector.setDesignated(myPdp,false);
 557                                                         isDesignated = false;
 558                                                         //If you can't promote it, demote it
 559                                                         try {
 560                                                                 String standbyStatus = stateManagementFeature.getStandbyStatus();
 561                                                                 if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 562                                                                                 standbyStatus.equals(StateManagement.COLD_STANDBY))){
 563                                                                         /*
 564                                                                          * Only call demote if it is not already in the right state.  Don't worry about
 565                                                                          * synching the lower level topic endpoint states.  That is done by the
 566                                                                          * refreshStateAudit.
 567                                                                          */
 568                                                                         stateManagementFeature.demote();
 569                                                                 }
 570                                                         } catch (Exception e1) {
 571                                                                 logger.error
 572                                                                 ("ERROR: DesignatedWaiter.run: Caught StandbyStatusException "
 573                                                                                 + "attempting to promote then demote PDP={}, message=",
 574                                                                                 myPdp.getPdpId(), e1);
 575                                                         }
 576
 577                                                 }
 578                                                 waitTimerLastRunDate = new Date();
 579                                                 if(logger.isDebugEnabled()){
 580                                                         logger.debug("DesignatedWaiter.run (designatedPdp.getPdpId().equals(myPdp.getPdpId())) "
 581                                                                 + "waitTimerLastRunDate = " + waitTimerLastRunDate);
 582                                                 }
 583
 584                                                 return;
 585                                         }
 586                                         isDesignated = false;
 587
 588                                 } // end synchronized
 589                                 if(logger.isDebugEnabled()){
 590                                         logger.debug
 591                                         ("DesignatedWaiter.run: myPdp: {}; Returning, isDesignated= {}",
 592                                                 isDesignated, myPdp.getPdpId());
 593                                 }
 594
 595                                 Date tmpDate = new Date();
 596                                 if(logger.isDebugEnabled()){
 597                                         logger.debug("DesignatedWaiter.run (end of run) waitTimerLastRunDate = {}", tmpDate);
 598                                 }
 599
 600                                 waitTimerLastRunDate = tmpDate;
 601
 602                         }catch(Exception e){
 603                                 logger.error("DesignatedWaiter.run caught an unexpected exception: ", e);
 604                         }
 605                 } // end run
 606         }
 607
 608         public ArrayList<DroolsPdp> santizeDesignatedList(ArrayList<DroolsPdp> listOfDesignated){
 609
 610                 boolean containsDesignated = false;
 611                 boolean containsHotStandby = false;
 612                 ArrayList<DroolsPdp> listForRemoval = new ArrayList<DroolsPdp>();
 613                 for(DroolsPdp pdp : listOfDesignated){
 614                         if(logger.isDebugEnabled()){
 615                                 logger.debug
 616                                 ("DesignatedWaiter.run sanitizing: pdp = {}"
 617                                         + " isDesignated = {}",pdp.getPdpId(), pdp.isDesignated());
 618                         }
 619                         if(pdp.isDesignated()){
 620                                 containsDesignated = true;
 621                         }else {
 622                                 containsHotStandby = true;
 623                                 listForRemoval.add(pdp);
 624                         }
 625                 }
 626                 if(containsDesignated && containsHotStandby){
 627                         //remove the hot standby from the list
 628                         listOfDesignated.removeAll(listForRemoval);
 629                         containsHotStandby = false;
 630                 }
 631                 return listOfDesignated;
 632         }
 633
 634         public DroolsPdp computeMostRecentPrimary(Collection<DroolsPdp> pdps, ArrayList<DroolsPdp> listOfDesignated){
 635                 boolean containsDesignated = false;
 636                 for(DroolsPdp pdp : listOfDesignated){
 637                         if(pdp.isDesignated()){
 638                                 containsDesignated = true;
 639                         }
 640                 }
 641                 DroolsPdp mostRecentPrimary = new DroolsPdpImpl(null, true, 1, new Date(0));
 642                 mostRecentPrimary.setSiteName(null);
 643                 if(logger.isDebugEnabled()){
 644                         logger.debug
 645                         ("DesignatedWaiter.run listOfDesignated.size() = {}", listOfDesignated.size());
 646                 }
 647                 if(listOfDesignated.size() <=1){
 648                         if(logger.isDebugEnabled()){
 649                                 logger.debug("DesignatedWainter.run: listOfDesignated.size <=1");
 650                         }
 651                         //Only one or none is designated or hot standby.  Choose the latest designated date
 652                         for(DroolsPdp pdp : pdps){
 653                                 if(logger.isDebugEnabled()){
 654                                         logger.debug
 655                                         ("DesignatedWaiter.run pdp = {}"
 656                                                 + " pdp.getDesignatedDate() = {}", pdp.getPdpId(), pdp.getDesignatedDate());
 657                                 }
 658                                 if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 659                                         mostRecentPrimary = pdp;
 660                                         if(logger.isDebugEnabled()){
 661                                                 logger.debug
 662                                                 ("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 663                                         }
 664                                 }
 665                         }
 666                 }else if(listOfDesignated.size() == pdps.size()){
 667                         if(logger.isDebugEnabled()){
 668                                 logger.debug("DesignatedWainter.run: listOfDesignated.size = pdps.size() which is {}", pdps.size());
 669                         }
 670                         //They are all designated or all hot standby.
 671                         mostRecentPrimary = null;
 672                         for(DroolsPdp pdp : pdps){
 673                                 if(mostRecentPrimary == null){
 674                                         mostRecentPrimary = pdp;
 675                                         continue;
 676                                 }
 677                                 if(containsDesignated){ //Choose the site of the first designated date
 678                                         if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) < 0){
 679                                                 mostRecentPrimary = pdp;
 680                                                 if(logger.isDebugEnabled()){
 681                                                         logger.debug
 682                                                         ("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 683                                                 }
 684                                         }
 685                                 }else{ //Choose the site with the latest designated date
 686                                         if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 687                                                 mostRecentPrimary = pdp;
 688                                                 if(logger.isDebugEnabled()){
 689                                                         logger.debug
 690                                                         ("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 691                                                 }
 692                                         }
 693                                 }
 694                         }
 695                 }else{
 696                         if(logger.isDebugEnabled()){
 697                                 logger.debug("DesignatedWainter.run: Some but not all are designated or hot standby. ");
 698                         }
 699                         //Some but not all are designated or hot standby.
 700                         if(containsDesignated){
 701                                 if(logger.isDebugEnabled()){
 702                                         logger.debug("DesignatedWainter.run: containsDesignated = {}", containsDesignated);
 703                                 }
 704                                 /*
 705                                  * The list only contains designated.  This is a problem.  It is most likely a race
 706                                  * condition that resulted in two thinking they should be designated. Choose the
 707                                  * site with the latest designated date for the pdp not included on the designated list.
 708                                  * This should be the site that had the last designation before this race condition
 709                                  * occurred.
 710                                  */
 711                                 for(DroolsPdp pdp : pdps){
 712                                         if(listOfDesignated.contains(pdp)){
 713                                                 continue; //Don't consider this entry
 714                                         }
 715                                         if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 716                                                 mostRecentPrimary = pdp;
 717                                                 if(logger.isDebugEnabled()){
 718                                                         logger.debug
 719                                                         ("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 720                                                 }
 721                                         }
 722                                 }
 723                         }else{
 724                                 if(logger.isDebugEnabled()){
 725                                         logger.debug("DesignatedWainter.run: containsDesignated = {}", containsDesignated);
 726                                 }
 727                                 //The list only contains hot standby. Choose the site of the latest designated date
 728                                 for(DroolsPdp pdp : pdps){
 729                                         if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 730                                                 mostRecentPrimary = pdp;
 731                                                 if(logger.isDebugEnabled()){
 732                                                         logger.debug
 733                                                         ("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 734                                                 }
 735                                         }
 736                                 }
 737                         }
 738                 }
 739                 return mostRecentPrimary;
 740         }
 741
 742         public DroolsPdp computeDesignatedPdp(ArrayList<DroolsPdp> listOfDesignated, DroolsPdp mostRecentPrimary){
 743                 DroolsPdp designatedPdp = null;
 744                 DroolsPdp lowestPriorityPdp = null;
 745                 if(listOfDesignated.size() > 1){
 746                         if(logger.isDebugEnabled()){
 747                                 logger.debug
 748                                 ("DesignatedWaiter.run: myPdp: {} listOfDesignated.size(): {}", myPdp.getPdpId(), listOfDesignated.size());
 749                         }
 750                         DroolsPdp rejectedPdp = null;
 751                         DroolsPdp lowestPrioritySameSite = null;
 752                         DroolsPdp lowestPriorityDifferentSite = null;
 753                         for(DroolsPdp pdp : listOfDesignated){
 754                                 // We need to determine if another PDP is the lowest priority
 755                                 if(nullSafeEquals(pdp.getSiteName(),mostRecentPrimary.getSiteName())){
 756                                         if(lowestPrioritySameSite == null){
 757                                                 if(lowestPriorityDifferentSite != null){
 758                                                         rejectedPdp = lowestPriorityDifferentSite;
 759                                                 }
 760                                                 lowestPrioritySameSite = pdp;
 761                                         }else{
 762                                                 if(pdp.getPdpId().equals((lowestPrioritySameSite.getPdpId()))){
 763                                                         continue;//nothing to compare
 764                                                 }
 765                                                 if(pdp.comparePriority(lowestPrioritySameSite) <0){
 766                                                         if(logger.isDebugEnabled()){
 767                                                                 logger.debug
 768                                                                 ("\nDesignatedWaiter.run: myPdp {}  listOfDesignated pdp ID: {}"
 769                                                                         + " has lower priority than pdp ID: {}",myPdp.getPdpId(), pdp.getPdpId(),
 770                                                                         lowestPrioritySameSite.getPdpId());
 771                                                         }
 772                                                         //we need to reject lowestPrioritySameSite
 773                                                         rejectedPdp = lowestPrioritySameSite;
 774                                                         lowestPrioritySameSite = pdp;
 775                                                 } else{
 776                                                         //we need to reject pdp and keep lowestPrioritySameSite
 777                                                         if(logger.isDebugEnabled()){
 778                                                                 logger.debug
 779                                                                 ("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {} "
 780                                                                         + " has higher priority than pdp ID: {}", myPdp.getPdpId(),pdp.getPdpId(),
 781                                                                         lowestPrioritySameSite.getPdpId());
 782                                                         }
 783                                                         rejectedPdp = pdp;
 784                                                 }
 785                                         }
 786                                 } else{
 787                                         if(lowestPrioritySameSite != null){
 788                                                 //if we already have a candidate for same site, we don't want to bother with different sites
 789                                                 rejectedPdp = pdp;
 790                                         } else{
 791                                                 if(lowestPriorityDifferentSite == null){
 792                                                         lowestPriorityDifferentSite = pdp;
 793                                                         continue;
 794                                                 }
 795                                                 if(pdp.getPdpId().equals((lowestPriorityDifferentSite.getPdpId()))){
 796                                                         continue;//nothing to compare
 797                                                 }
 798                                                 if(pdp.comparePriority(lowestPriorityDifferentSite) <0){
 799                                                         if(logger.isDebugEnabled()){
 800                                                                 logger.debug
 801                                                                 ("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 802                                                                         + " has lower priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 803                                                                         lowestPriorityDifferentSite.getPdpId());
 804                                                         }
 805                                                         //we need to reject lowestPriorityDifferentSite
 806                                                         rejectedPdp = lowestPriorityDifferentSite;
 807                                                         lowestPriorityDifferentSite = pdp;
 808                                                 } else{
 809                                                         //we need to reject pdp and keep lowestPriorityDifferentSite
 810                                                         if(logger.isDebugEnabled()){
 811                                                                 logger.debug
 812                                                                 ("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 813                                                                         + " has higher priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 814                                                                         lowestPriorityDifferentSite.getPdpId());
 815                                                         }
 816                                                         rejectedPdp = pdp;
 817                                                 }
 818                                         }
 819                                 }
 820                                 // If the rejectedPdp is myPdp, we need to stand it down and demote it.  Each pdp is responsible
 821                                 // for demoting itself
 822                                 if(rejectedPdp != null && nullSafeEquals(rejectedPdp.getPdpId(),myPdp.getPdpId())){
 823                                         if(logger.isDebugEnabled()){
 824                                                 logger.debug
 825                                                 ("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated myPdp ID: {}"
 826                                                         + " is NOT the lowest priority.  Executing stateManagement.demote()\n\n", myPdp.getPdpId(),
 827                                                          myPdp.getPdpId());
 828                                         }
 829                                         // We found that myPdp is on the listOfDesignated and it is not the lowest priority
 830                                         // So, we must demote it
 831                                         try {
 832                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 833                                                 myPdp.setDesignated(false);
 834                                                 pdpsConnector.setDesignated(myPdp, false);
 835                                                 isDesignated = false;
 836                                                 String standbyStatus = stateManagementFeature.getStandbyStatus();
 837                                                 if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 838                                                                 standbyStatus.equals(StateManagement.COLD_STANDBY))){
 839                                                         /*
 840                                                          * Only call demote if it is not already in the right state.  Don't worry about
 841                                                          * synching the lower level topic endpoint states.  That is done by the
 842                                                          * refreshStateAudit.
 843                                                          */
 844                                                         stateManagementFeature.demote();
 845                                                 }
 846                                         } catch (Exception e) {
 847                                                 myPdp.setDesignated(false);
 848                                                 pdpsConnector.setDesignated(myPdp, false);
 849                                                 isDesignated = false;
 850                                                 logger.error
 851                                                 ("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 852                                                                 + "demote myPdp {} myPdp.getPdpId(), message= {}", myPdp.getPdpId(),
 853                                                                 e);
 854                                         }
 855                                 }
 856                         } //end: for(DroolsPdp pdp : listOfDesignated)
 857                         if(lowestPrioritySameSite != null){
 858                                 lowestPriorityPdp = lowestPrioritySameSite;
 859                         } else {
 860                                 lowestPriorityPdp = lowestPriorityDifferentSite;
 861                         }
 862                         //now we have a valid value for lowestPriorityPdp
 863                         if(logger.isDebugEnabled()){
 864                                 logger.debug
 865                                 ("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 866                                         + "found the LOWEST priority pdp ID: {} "
 867                                         + " It is now the designatedPpd from the perspective of myPdp ID: {} \n\n",
 868                                          myPdp.getPdpId(), lowestPriorityPdp.getPdpId(), myPdp);
 869                         }
 870                         designatedPdp = lowestPriorityPdp;
 871
 872                 } else if(listOfDesignated.isEmpty()){
 873                         if(logger.isDebugEnabled()){
 874                                 logger.debug
 875                                 ("\nDesignatedWaiter.run: myPdp: {} listOfDesignated is: EMPTY.", myPdp.getPdpId());
 876                         }
 877                         designatedPdp = null;
 878                 } else{ //only one in listOfDesignated
 879                         if(logger.isDebugEnabled()){
 880                                 logger.debug
 881                                 ("\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 882                                         + "has ONE entry. PDP ID: {}", myPdp.getPdpId(), listOfDesignated.get(0).getPdpId());
 883                         }
 884                         designatedPdp = listOfDesignated.get(0);
 885                 }
 886                 return designatedPdp;
 887
 888         }
 889
 890         private class TimerUpdateClass extends TimerTask{
 891
 892                 @Override
 893                 public void run() {
 894                         try{
 895                                 if(logger.isDebugEnabled()){
 896                                         logger.debug("TimerUpdateClass.run: entry");
 897                                 }
 898                                 checkWaitTimer();
 899                                 synchronized(pdpsConnectorLock){
 900
 901                                         myPdp.setUpdatedDate(new Date());
 902                                         /*
 903                                         Redundant with DesignationWaiter and this updates the date every
 904                                         cycle instead of just when the state changes.
 905                                         if(myPdp.isDesignated()){
 906                                                 myPdp.setDesignatedDate(new Date());
 907                                         }
 908                                         */
 909                                         pdpsConnector.update(myPdp);
 910
 911                                         Date tmpDate = new Date();
 912                                         if(logger.isDebugEnabled()){
 913                                                 logger.debug("TimerUpdateClass.run: updateWorkerLastRunDate = {}", tmpDate);
 914                                         }
 915
 916                                         updateWorkerLastRunDate = tmpDate;
 917                                 }
 918                                 if(logger.isDebugEnabled()){
 919                                         logger.debug("TimerUpdateClass.run.exit");
 920                                 }
 921                         }catch(Exception e){
 922                                 logger.error("TimerUpdateClass.run caught an unexpected exception: ", e);
 923                         }
 924                 }
 925         }
 926         @Override
 927         public void checkThreadStatus() {
 928                 checkUpdateWorkerTimer();
 929                 checkWaitTimer();
 930         }
 931
 932         private void checkUpdateWorkerTimer(){
 933                 synchronized(checkUpdateWorkerLock){
 934                         try{
 935                                 if(logger.isDebugEnabled()){
 936                                         logger.debug("checkUpdateWorkerTimer: entry");
 937                                 }
 938                                 Date now = new Date();
 939                                 long nowMs = now.getTime();
 940                                 long updateWorkerMs = updateWorkerLastRunDate.getTime();
 941                                 //give it 2 second cushion
 942                                 if((nowMs - updateWorkerMs)  > pdpCheckInterval + 2000){
 943                                         logger.error("checkUpdateWorkerTimer: nowMs - updateWorkerMs = {} "
 944                                                         + ", exceeds pdpCheckInterval + 2000 = {} "
 945                                                         + "Will reschedule updateWorker timer",(nowMs - updateWorkerMs), (pdpCheckInterval + 2000));
 946
 947                                         try{
 948                                                 updateWorker.cancel();
 949                                                 // Recalculate the time because this is a synchronized section and the thread could have
 950                                                 // been blocked.
 951                                                 now = new Date();
 952                                                 nowMs = now.getTime();
 953                                                 updateWorker = new Timer();
 954                                                 // reset the updateWorkerLastRunDate
 955                                                 updateWorkerLastRunDate = new Date(nowMs + 100);
 956                                                 //execute the first time in 100 ms
 957                                                 updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
 958                                                 if(logger.isDebugEnabled()){
 959                                                         logger.debug("checkUpdateWorkerTimer: Scheduling updateWorker timer to start in 100 ms ");
 960                                                 }
 961                                         }catch(Exception e){
 962                                                 logger.error("checkUpdateWorkerTimer: Caught unexpected Exception: ", e);
 963                                                 // Recalculate the time because this is a synchronized section and the thread could have
 964                                                 // been blocked.
 965                                                 now = new Date();
 966                                                 nowMs = now.getTime();
 967                                                 updateWorker = new Timer();
 968                                                 updateWorkerLastRunDate = new Date(nowMs + 100);
 969                                                 updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
 970                                                 if(logger.isDebugEnabled()){
 971                                                         logger.debug("checkUpdateWorkerTimer: Attempting to schedule updateWorker timer in 100 ms");
 972                                                 }
 973                                         }
 974
 975                                 }
 976                                 if(logger.isDebugEnabled()){
 977                                         logger.debug("checkUpdateWorkerTimer: exit");
 978                                 }
 979                         }catch(Exception e){
 980                                 logger.error("checkUpdateWorkerTimer: caught unexpected exception: ", e);
 981                         }
 982                 }
 983         }
 984
 985         private void checkWaitTimer(){
 986                 synchronized(checkWaitTimerLock){
 987                         try{
 988                                 if(logger.isDebugEnabled()){
 989                                         logger.debug("checkWaitTimer: entry");
 990                                 }
 991                                 Date now = new Date();
 992                                 long nowMs = now.getTime();
 993                                 long waitTimerMs = waitTimerLastRunDate.getTime();
 994
 995                                 //give it 2 times leeway
 996                                 if((nowMs - waitTimerMs)  > 2*pdpUpdateInterval){
 997                                         logger.error("checkWaitTimer: nowMs - waitTimerMs = {}"
 998                                                         + ", exceeds pdpUpdateInterval + 2000 = {}"
 999                                                         + "Will reschedule waitTimer timer", (nowMs - waitTimerMs), (2*pdpUpdateInterval));
1000
1001                                         try{
1002                                                 // Recalculate since the thread could have been stalled on the synchronize()
1003                                                 nowMs = (new Date()).getTime();
1004                                                 // Time to the start of the next pdpUpdateInterval multiple
1005                                                 long startMs = getDWaiterStartMs();
1006                                                 waitTimer.cancel();
1007                                                 designationWaiter = new DesignationWaiter();
1008                                                 waitTimer = new Timer();
1009                                                 waitTimerLastRunDate = new Date(nowMs + startMs);
1010                                                 waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
1011                                                 if(logger.isDebugEnabled()){
1012                                                         logger.debug("checkWaitTimer: Scheduling waitTimer timer to start in {} ms", startMs);
1013                                                 }
1014                                         }catch(Exception e){
1015                                                 logger.error("checkWaitTimer: Caught unexpected Exception: ", e);
1016                                                 // Recalculate since the thread could have been stalled on the synchronize()
1017                                                 nowMs = (new Date()).getTime();
1018                                                 // Time to the start of the next pdpUpdateInterval multiple
1019                                                 long startMs = getDWaiterStartMs();
1020                                                 designationWaiter = new DesignationWaiter();
1021                                                 waitTimer = new Timer();
1022                                                 waitTimerLastRunDate = new Date(nowMs + startMs);
1023                                                 waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
1024                                                 if(logger.isDebugEnabled()){
1025                                                         logger.debug("checkWaitTimer: Scheduling waitTimer timer in {} ms", startMs);
1026                                                 }
1027                                         }
1028
1029                                 }
1030                                 if(logger.isDebugEnabled()){
1031                                         logger.debug("checkWaitTimer: exit");
1032                                 }
1033                         }catch(Exception e){
1034                                 logger.error("checkWaitTimer: caught unexpected exception: ", e);
1035                         }
1036                 }
1037         }
1038
1039         private long getDWaiterStartMs(){
1040                 Date now = new Date();
1041
1042                 // Retrieve the ms since the epoch
1043                 long nowMs = now.getTime();
1044
1045                 // Time since the end of the last pdpUpdateInterval multiple
1046                 long nowModMs = nowMs % pdpUpdateInterval;
1047
1048                 // Time to the start of the next pdpUpdateInterval multiple
1049                 long startMs = 2*pdpUpdateInterval - nowModMs;
1050
1051                 // Give the start time a minimum of a 5 second cushion
1052                 if(startMs < 5000){
1053                         // Start at the beginning  of following interval
1054                         startMs = pdpUpdateInterval + startMs;
1055                 }
1056                 return startMs;
1057         }
1058
1059         private boolean nullSafeEquals(Object one, Object two){
1060                 if(one == null && two == null){
1061                         return true;
1062                 }
1063                 if(one != null && two != null){
1064                         return one.equals(two);
1065                 }
1066                 return false;
1067         }
1068
1069         public String getPdpdNowActive(){
1070                 return pdpdNowActive;
1071         }
1072
1073         public String getPdpdLastActive(){
1074                 return pdpdLastActive;
1075         }
1076 }