policy-persistence/src/main/java/org/openecomp/policy/drools/persistence/DroolsPdpsElectionHandler.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * policy-persistence
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.policy.drools.persistence;
  22
  23 import java.util.ArrayList;
  24 import java.util.Collection;
  25 import java.util.Date;
  26 import java.util.LinkedList;
  27 import java.util.List;
  28 import java.util.Timer;
  29 import java.util.TimerTask;
  30
  31 import org.openecomp.policy.common.im.StandbyStatusException;
  32 import org.openecomp.policy.common.im.StateManagement;
  33 import org.openecomp.policy.drools.core.DroolsPDPIntegrityMonitor;
  34 import org.openecomp.policy.drools.core.IntegrityMonitorProperties;
  35 import org.openecomp.policy.common.logging.flexlogger.FlexLogger;
  36 import org.openecomp.policy.common.logging.flexlogger.Logger;
  37 import org.openecomp.policy.common.logging.eelf.MessageCodes;
  38
  39 public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
  40         // get an instance of logger
  41         private final static Logger  logger = FlexLogger.getLogger(DroolsPdpsElectionHandler.class);
  42         private DroolsPdpsConnector pdpsConnector;
  43         private Object pdpsConnectorLock = new Object();
  44         private Object checkUpdateWorkerLock = new Object();
  45         private Object checkWaitTimerLock = new Object();
  46         private Object designationWaiterLock = new Object();
  47
  48         /*
  49          * Must be static, so it can be referenced by JpaDroolsPdpsConnector,
  50          * without requiring a reference to the election handler instantiation.
  51          */
  52         private static DroolsPdp myPdp;
  53
  54         private DesignationWaiter designationWaiter;
  55         private Timer updateWorker;
  56         private Timer waitTimer;
  57         private Date updateWorkerLastRunDate;
  58         private Date waitTimerLastRunDate;
  59         private int pdpCheckInterval;
  60         private int pdpUpdateInterval;
  61         private volatile boolean isDesignated;
  62         DroolsPDPIntegrityMonitor droolsPdpIntegrityMonitor;
  63         StateManagement stateManagement;
  64
  65         public DroolsPdpsElectionHandler(DroolsPdpsConnector pdps, DroolsPdp myPdp, DroolsPDPIntegrityMonitor droolsPdpIntegrityMonitor){
  66                 this.pdpsConnector = pdps;
  67                 DroolsPdpsElectionHandler.myPdp = myPdp;
  68                 this.isDesignated = false;
  69                 this.droolsPdpIntegrityMonitor = droolsPdpIntegrityMonitor;
  70                 this.stateManagement = droolsPdpIntegrityMonitor.getStateManager();
  71                 pdpCheckInterval = 3000;
  72                 try{
  73                         pdpCheckInterval = Integer.parseInt(IntegrityMonitorProperties.getProperty(IntegrityMonitorProperties.PDP_CHECK_INVERVAL));
  74                 }catch(Exception e){
  75                         logger.error
  76                         //System.out.println
  77                         (MessageCodes.EXCEPTION_ERROR ,e, "Could not get pdpCheckInterval property. Using default");
  78                 }
  79                 pdpUpdateInterval = 2000;
  80                 try{
  81                         pdpUpdateInterval = Integer.parseInt(IntegrityMonitorProperties.getProperty(IntegrityMonitorProperties.PDP_UPDATE_INTERVAL));
  82                 }catch(Exception e){
  83                         logger.error
  84                         //System.out.println
  85                         (MessageCodes.EXCEPTION_ERROR, e, "Could not get pdpUpdateInterval property. Using default");
  86                 }
  87
  88                 Date now = new Date();
  89
  90                 // Retrieve the ms since the epoch
  91                 long nowMs = now.getTime();
  92
  93                 // Create the timer which will update the updateDate in DroolsPdpEntity table.
  94                 // This is the heartbeat
  95                 updateWorker = new Timer();
  96
  97                 // Schedule the heartbeat to start in 100 ms and run at pdpCheckInterval ms thereafter
  98                 updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
  99                 updateWorkerLastRunDate = new Date(nowMs + 100);
 100
 101                 // Create the timer which will run the election algorithm
 102                 waitTimer = new Timer();
 103
 104                 // Schedule it to start in startMs ms (so it will run after the updateWorker and run at pdpUpdateInterval ms thereafter
 105                 long startMs = getDWaiterStartMs();
 106                 designationWaiter = new DesignationWaiter();
 107                 waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
 108                 waitTimerLastRunDate = new Date(nowMs + startMs);
 109         }
 110
 111         public List<DroolsSessionEntity> waitForDesignation(){
 112                 while(isDesignated == false){
 113                         try {
 114                                 Thread.sleep(1000);
 115                         } catch (InterruptedException e) {
 116                                 return null;
 117                         }
 118                 }
 119                 return designationWaiter.getSessions();
 120
 121         }
 122         public List<DroolsSessionEntity> getSessions(){
 123                 return designationWaiter.getSessions();
 124         }
 125         public void updateMyPdp(){
 126                 synchronized(pdpsConnectorLock){
 127                         myPdp.setUpdatedDate(new Date());
 128                         pdpsConnector.update(myPdp);
 129                 }
 130         }
 131
 132         /*
 133          * When the JpaDroolsPdpsConnector.standDown() method is invoked, it needs
 134          * access to myPdp, so it can keep its designation status in sync with the
 135          * DB.
 136          */
 137         public static void setMyPdpDesignated(boolean designated) {
 138                 logger.debug
 139                 //System.out.println
 140                         ("setMyPdpDesignated: designated=" + designated);
 141                 myPdp.setDesignated(designated);
 142         }
 143
 144         private class DesignationWaiter extends TimerTask {
 145                 // get an instance of logger
 146                 private Logger  logger = FlexLogger.getLogger(DesignationWaiter.class);
 147                 private List<DroolsSessionEntity> sessions = null;
 148
 149                 public List<DroolsSessionEntity> getSessions(){
 150                         if(sessions != null){
 151                                 return sessions;
 152                         }
 153                         return new LinkedList<DroolsSessionEntity>();
 154                 }
 155                 public void run() {
 156                         try{
 157                                 logger.debug
 158                                 //System.out.println
 159                                 ("DesignatedWaiter.run: Entering");
 160
 161                                 // just here initially so code still works
 162                                 if (pdpsConnector == null) {
 163                                         waitTimerLastRunDate = new Date();
 164                                         logger.info("DesignatedWaiter.run (pdpsConnector==null) waitTimerLastRunDate = " + waitTimerLastRunDate);
 165
 166                                         return;
 167                                 }
 168
 169                                 synchronized (designationWaiterLock) {
 170
 171                                         logger.debug
 172                                         //System.out.println
 173                                         ("DesignatedWaiter.run: Entering synchronized block");
 174
 175                                         checkUpdateWorkerTimer();
 176
 177                                         //It is possible that multiple PDPs are designated lead.  So, we will make a list of all designated
 178                                         //PDPs and then decide which one really should be designated at the end.
 179                                         ArrayList<DroolsPdp> listOfDesignated = new ArrayList<DroolsPdp>();
 180
 181                                         Collection<DroolsPdp> pdps = pdpsConnector.getDroolsPdps();
 182                                         DroolsPdp designatedPdp = null;
 183
 184                                         logger.debug
 185                                         //System.out.println
 186                                         ("DesignatedWaiter.run: pdps.size="
 187                                                         + pdps.size());
 188
 189                                         //This is only true if all designated PDPs have failed
 190                                         boolean designatedPdpHasFailed = pdpsConnector.hasDesignatedPdpFailed(pdps);
 191                                         logger.debug
 192                                         //System.out.println
 193                                         ("DesignatedWaiter.run: designatedPdpHasFailed="
 194                                                         + designatedPdpHasFailed);
 195                                         for (DroolsPdp pdp : pdps) {
 196                                                 logger.debug
 197                                                 //System.out.println
 198                                                 ("DesignatedWaiter.run: evaluating pdp ID: " + pdp.getPdpId());
 199
 200                                                 /*
 201                                                  * Note: side effect of isPdpCurrent is that any stale but
 202                                                  * designated PDPs will be marked as un-designated.
 203                                                  */
 204                                                 boolean isCurrent = pdpsConnector.isPdpCurrent(pdp);
 205
 206                                                 /*
 207                                                  * We can't use stateManagement.getStandbyStatus() here, because
 208                                                  * we need the standbyStatus, not for this PDP, but for the PDP
 209                                                  * being processed by this loop iteration.
 210                                                  */
 211                                                 String standbyStatus = stateManagement.getStandbyStatus(pdp.getPdpId());
 212                                                 if(standbyStatus==null){
 213                                                         // Treat this case as a cold standby -- if we
 214                                                         // abort here, no sessions will be created in a
 215                                                         // single-node test environment.
 216                                                         standbyStatus = StateManagement.COLD_STANDBY;
 217                                                 }
 218
 219                                                 logger.debug
 220                                                 //System.out.println
 221                                                 ("DesignatedWaiter.run: PDP="
 222                                                                 + pdp.getPdpId() + ", isCurrent=" + isCurrent);
 223
 224                                                 /*
 225                                                  * There are 4 combinations of isDesignated and isCurrent.  We will examine each one in-turn
 226                                                  * and evaluate the each pdp in the list of pdps against each combination.
 227                                                  *
 228                                                  * This is the first combination of isDesignated and isCurrent
 229                                                  */
 230                                                 if (pdp.isDesignated()  &&  isCurrent) {
 231                                                         //It is current, but it could have a standbystatus=coldstandby / hotstandby
 232                                                         //If so, we need to stand it down and demote it
 233                                                         if(!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)){
 234                                                                 if(pdp.getPdpId().equals(myPdp.getPdpId())){
 235                                                                         logger.debug
 236                                                                         //System.out.println
 237                                                                         ("\n\nDesignatedWaiter.run: myPdp " + myPdp.getPdpId() + " is current and designated, "
 238                                                                                         + "butstandbystatus is not providingservice. "
 239                                                                                         + " Executing stateManagement.demote()" + "\n\n");
 240                                                                         // So, we must demote it
 241                                                                         try {
 242                                                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 243                                                                                 //This will change isDesignated and it can enter another if(combination) below
 244                                                                                 pdpsConnector.standDownPdp(pdp.getPdpId());
 245                                                                                 myPdp.setDesignated(false);
 246                                                                                 isDesignated = false;
 247                                                                                 if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 248                                                                                                 standbyStatus.equals(StateManagement.COLD_STANDBY))){
 249                                                                                         /*
 250                                                                                          * Only demote it if it appears it has not already been demoted. Don't worry
 251                                                                                          * about synching with the topic endpoint states.  That is done by the
 252                                                                                          * refreshStateAudit
 253                                                                                          */
 254                                                                                         stateManagement.demote();
 255                                                                                 }
 256                                                                                 //update the standbystatus to check in a later combination of isDesignated and isCurrent
 257                                                                                 standbyStatus=stateManagement.getStandbyStatus(pdp.getPdpId());
 258                                                                         } catch (Exception e) {
 259                                                                                 logger.error
 260                                                                                 //System.out.println
 261                                                                                 ("DesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " Caught Exception attempting to demote myPdp'"
 262                                                                                                 + myPdp.getPdpId()
 263                                                                                                 + "', message="
 264                                                                                                 + e.getMessage());
 265                                                                                 System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 266                                                                                                 + "from stateManagement.demote()");
 267                                                                                 e.printStackTrace();
 268                                                                         }
 269                                                                 }else{
 270                                                                         // Don't demote a remote PDP that is current.  It should catch itself
 271                                                                         logger.debug
 272                                                                         //System.out.println
 273                                                                         ("\n\nDesignatedWaiter.run: myPdp " + myPdp.getPdpId() + " is current and designated, "
 274                                                                                         + "but standbystatus is not providingservice. "
 275                                                                                         + " Cannot execute stateManagement.demote() since it it is not myPdp\n\n");
 276                                                                 }
 277
 278                                                         }else{
 279                                                                 // If we get here, it is ok to be on the list
 280                                                                 logger.debug
 281                                                                 //System.out.println
 282                                                                 ("DesignatedWaiter.run: PDP="
 283                                                                                 + pdp.getPdpId()
 284                                                                                 + " is designated, current and " + standbyStatus +".  Noting PDP as designated.  standbyStatus=" + standbyStatus);
 285                                                                 listOfDesignated.add(pdp);
 286                                                         }
 287
 288
 289                                                 }
 290
 291
 292                                                 /*
 293                                                  * The second combination of isDesignated and isCurrent
 294                                                  *
 295                                                  * PDP is designated but not current; it has failed.   So we stand it down (it doesn't matter what
 296                                                  * its standbyStatus is). None of these go on the list.
 297                                                  */
 298                                                 if (pdp.isDesignated()  &&  !isCurrent) {
 299                                                         logger.info
 300                                                         //System.out.println
 301                                                         ("INFO: DesignatedWaiter.run: PDP="
 302                                                                         + pdp.getPdpId()
 303                                                                         + " is currently designated but is not current; it has failed.  Standing down.  standbyStatus=" + standbyStatus);
 304
 305                                                         /*
 306                                                          * Changes designated to 0 but it is still potentially providing service
 307                                                          * Will affect isDesignated, so, it can enter an if(combination) below
 308                                                          */
 309                                                         pdpsConnector.standDownPdp(pdp.getPdpId());
 310
 311                                                         //need to change standbystatus to coldstandby
 312                                                         if (pdp.getPdpId().equals(myPdp.getPdpId())){
 313                                                                 logger.debug
 314                                                                 //System.out.println
 315                                                                 ("\n\nDesignatedWaiter.run: myPdp " + myPdp.getPdpId() + " is not Current. "
 316                                                                                 + " Executing stateManagement.disableFailed()" + "\n\n");
 317                                                                 // We found that myPdp is designated but not current
 318                                                                 // So, we must cause it to disableFail
 319                                                                 try {
 320                                                                         myPdp.setDesignated(false);
 321                                                                         //pdpsConnector.setDesignated(myPdp, false);//not needed?
 322                                                                         isDesignated = false;
 323                                                                         stateManagement.disableFailed();
 324                                                                         //stateManagement.demote();
 325                                                                 } catch (Exception e) {
 326                                                                         logger.error
 327                                                                         //System.out.println
 328                                                                         ("DesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " Caught Exception attempting to disableFail myPdp'"
 329                                                                                         + myPdp.getPdpId()
 330                                                                                         + "', message="
 331                                                                                         + e.getMessage());
 332                                                                         System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 333                                                                                         + "from stateManagement.disableFailed()");
 334                                                                         e.printStackTrace();
 335                                                                 }
 336                                                         } else { //it is a remote PDP that is failed
 337                                                                 logger.debug
 338                                                                 //System.out.println
 339                                                                 ("\n\nDesignatedWaiter.run: PDP " + pdp.getPdpId() + " is not Current. "
 340                                                                                 + " Executing stateManagement.disableFailed(otherResourceName)" + "\n\n");
 341                                                                 // We found a PDP is designated but not current
 342                                                                 // We already called standdown(pdp) which will change designated to false
 343                                                                 // Now we need to disableFail it to get its states in synch.  The standbyStatus
 344                                                                 // should equal coldstandby
 345                                                                 try {
 346                                                                         stateManagement.disableFailed(pdp.getPdpId());
 347                                                                         //stateManagement.demote(pdp.getPdpId());
 348                                                                 } catch (Exception e) {
 349                                                                         logger.error
 350                                                                         //System.out.println
 351                                                                         ("DesignatedWaiter.run: for PDP" + pdp.getPdpId()
 352                                                                                         + " Caught Exception attempting to disableFail(" + pdp.getPdpId() + ")'"
 353                                                                                         + pdp.getPdpId()
 354                                                                                         + "', message="
 355                                                                                         + e.getMessage());
 356                                                                         System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 357                                                                                         + "from stateManagement.disableFailed()");
 358                                                                         e.printStackTrace();
 359                                                                 }
 360
 361                                                         }
 362                                                         continue; //we are not going to do anything else with this pdp
 363                                                 }
 364
 365                                                 /*
 366                                                  * The third combination of isDesignated and isCurrent
 367                                                  * /*
 368                                                  * If a PDP is not currently designated but is providing service (erroneous, but recoverable) or hot standby
 369                                                  * we can add it to the list of possible designated if all the designated have failed
 370                                                  */
 371                                                 if (!pdp.isDesignated() && isCurrent){
 372                                                         if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 373                                                                         standbyStatus.equals(StateManagement.COLD_STANDBY))){
 374                                                                 logger.info("\n\nDesignatedWaiter.run: PDP " + pdp.getPdpId()
 375                                                                                 + " is NOT designated but IS current and"
 376                                                                                 + " has a standbystatus=" + standbyStatus);
 377                                                                 // Since it is current, we assume it can adjust its own state.
 378                                                                 // We will demote if it is myPdp
 379                                                                 if(pdp.getPdpId().equals(myPdp.getPdpId())){
 380                                                                         //demote it
 381                                                                         logger.info("DesignatedWaiter.run: PDP " + pdp.getPdpId() + " going to "
 382                                                                                         + "setDesignated = false and calling stateManagement.demote");
 383                                                                         try {
 384                                                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 385                                                                                 pdpsConnector.setDesignated(myPdp, false);
 386                                                                                 myPdp.setDesignated(false);
 387                                                                                 isDesignated = false;
 388                                                                                 //This is definitely not a redundant call.  It is attempting to correct a problem
 389                                                                                 stateManagement.demote();
 390                                                                                 //recheck the standbystatus
 391                                                                                 standbyStatus = stateManagement.getStandbyStatus(pdp.getPdpId());
 392                                                                         } catch (Exception e) {
 393                                                                                 logger.error
 394                                                                                 //System.out.println
 395                                                                                 ("DesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " Caught Exception attempting to demote myPdp'"
 396                                                                                                 + myPdp.getPdpId()
 397                                                                                                 + "', message="
 398                                                                                                 + e.getMessage());
 399                                                                                 System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 400                                                                                                 + "from stateManagement.demote()");
 401                                                                                 e.printStackTrace();
 402                                                                         }
 403
 404                                                                 }
 405                                                         }
 406                                                         if(standbyStatus.equals(StateManagement.HOT_STANDBY) && designatedPdpHasFailed){
 407                                                                 //add it to the list
 408                                                                 logger.info
 409                                                                 //System.out.println
 410                                                                 ("INFO: DesignatedWaiter.run: PDP=" + pdp.getPdpId()
 411                                                                                 + " is not designated but is " + standbyStatus + " and designated PDP has failed.  standbyStatus="
 412                                                                                 + standbyStatus);
 413                                                                 logger.info
 414                                                                 //System.out.println
 415                                                                 ("DesignatedWaiter.run: Designating PDP=" + pdp.getPdpId());
 416                                                                 listOfDesignated.add(pdp);
 417                                                         }
 418                                                         continue; //done with this one
 419                                                 }
 420
 421                                                 /*
 422                                                  * The fourth combination of isDesignated and isCurrent
 423                                                  *
 424                                                  * We are not going to put any of these on the list since it appears they have failed.
 425
 426                                                  *
 427                                                  */
 428                                                 if(!pdp.isDesignated() && !isCurrent) {
 429                                                         logger.info
 430                                                         //System.out.println
 431                                                         ("INFO: DesignatedWaiter.run: PDP="
 432                                                                         + pdp.getPdpId() + ", designated="
 433                                                                         + pdp.isDesignated() + ", current="
 434                                                                         + isCurrent
 435                                                                         + ", designatedPdpHasFailed="
 436                                                                         + designatedPdpHasFailed
 437                                                                         + ",  standbyStatus=" + standbyStatus);
 438                                                         if(!standbyStatus.equals(StateManagement.COLD_STANDBY)){
 439                                                                 //stand it down
 440                                                                 //disableFail it
 441                                                                 pdpsConnector.standDownPdp(pdp.getPdpId());
 442                                                                 if(pdp.getPdpId().equals(myPdp.getPdpId())){
 443                                                                         /*
 444                                                                          * I don't actually know how this condition could happen, but if it did, we would want
 445                                                                          * to declare it failed.
 446                                                                          */
 447                                                                         logger.debug
 448                                                                         //System.out.println
 449                                                                         ("\n\nDesignatedWaiter.run: myPdp " + myPdp.getPdpId() + " is !current and !designated, "
 450                                                                                         + " Executing stateManagement.disableFailed()" + "\n\n");
 451                                                                         // So, we must disableFail it
 452                                                                         try {
 453                                                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 454                                                                                 myPdp.setDesignated(false);
 455                                                                                 isDesignated = false;
 456                                                                                 stateManagement.disableFailed();
 457                                                                                 //stateManagement.demote();
 458                                                                         } catch (Exception e) {
 459                                                                                 logger.error
 460                                                                                 //System.out.println
 461                                                                                 ("DesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " Caught Exception attempting to disableFail myPdp'"
 462                                                                                                 + myPdp.getPdpId()
 463                                                                                                 + "', message="
 464                                                                                                 + e.getMessage());
 465                                                                                 System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 466                                                                                                 + "from stateManagement.disableFailed()");
 467                                                                                 e.printStackTrace();
 468                                                                         }
 469                                                                 }else{//it is remote
 470                                                                         logger.debug
 471                                                                         //System.out.println
 472                                                                         ("\n\nDesignatedWaiter.run: myPdp " + myPdp.getPdpId() + " is !current and !designated, "
 473                                                                                         + " Executing stateManagement.disableFailed(" + pdp.getPdpId() + ")" + "\n\n");
 474                                                                         // We already called standdown(pdp) which will change designated to false
 475                                                                         // Now we need to disableFail it to get its states in sync.  StandbyStatus = coldstandby
 476                                                                         try {
 477                                                                                 stateManagement.disableFailed(pdp.getPdpId());
 478                                                                                 //stateManagement.demote(pdp.getPdpId());
 479                                                                         } catch (Exception e) {
 480                                                                                 logger.error
 481                                                                                 //System.out.println
 482                                                                                 ("DesignatedWaiter.run: for PDP" + pdp.getPdpId()
 483                                                                                                 + " Caught Exception attempting to disableFail(" + pdp.getPdpId() + ")'"
 484                                                                                                 + pdp.getPdpId()
 485                                                                                                 + "', message="
 486                                                                                                 + e.getMessage());
 487                                                                                 System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 488                                                                                                 + "from stateManagement.disableFailed()");
 489                                                                                 e.printStackTrace();
 490                                                                         }
 491                                                                 }
 492                                                         }
 493                                                 }
 494
 495
 496                                         } // end pdps loop
 497
 498                                         /*
 499                                          * We have checked the four combinations of isDesignated and isCurrent.  Where appropriate,
 500                                          * we added the PDPs to the potential list of designated pdps
 501                                          *
 502                                          * We need to give priority to pdps on the same site that is currently being used
 503                                          * First, however, we must sanitize the list of designated to make sure their are
 504                                          * only designated members or non-designated members.  There should not be both in
 505                                          * the list. Because there are real time delays, it is possible that both types could
 506                                          * be on the list.
 507                                          */
 508
 509                                         listOfDesignated = santizeDesignatedList(listOfDesignated);
 510
 511                                         /*
 512                                          * We need to figure out the last pdp that was the primary so we can get the last site
 513                                          * name and the last session numbers.  We need to create a "dummy" droolspdp since
 514                                          * it will be used in later comparrisons and cannot be null.
 515                                          */
 516
 517                                         DroolsPdp mostRecentPrimary = computeMostRecentPrimary(pdps, listOfDesignated);
 518
 519
 520                                         /*
 521                                          * It is possible to get here with more than one pdp designated and providingservice. This normally
 522                                          * occurs when there is a race condition with multiple nodes coming up at the same time. If that is
 523                                          * the case we must determine which one is the one that should be designated and which one should
 524                                          * be demoted.
 525                                          *
 526                                          * It is possible to have 0, 1, 2 or more but not all, or all designated.
 527                                          *   If we have one designated and current, we chose it and are done
 528                                          *   If we have 2 or more, but not all, we must determine which one is in the same site as
 529                                          *   the previously designated pdp.
 530                                          */
 531
 532                                         designatedPdp = computeDesignatedPdp(listOfDesignated, mostRecentPrimary);
 533
 534
 535                                         if (designatedPdp == null) {
 536                                                 logger.warn
 537                                                 //System.out.println
 538                                                 ("WARNING: DesignatedWaiter.run: No viable PDP found to be Designated. designatedPdp still null.");
 539                                                 // Just to be sure the parameters are correctly set
 540                                                 myPdp.setDesignated(false);
 541                                                 pdpsConnector.setDesignated(myPdp,false);
 542                                                 isDesignated = false;
 543
 544                                                 waitTimerLastRunDate = new Date();
 545                                                 logger.info("DesignatedWaiter.run (designatedPdp == null) waitTimerLastRunDate = " + waitTimerLastRunDate);
 546
 547                                                 return;
 548
 549                                         } else if (designatedPdp.getPdpId().equals(myPdp.getPdpId())) {
 550                                                 logger.debug
 551                                                 //System.out.println
 552                                                 ("DesignatedWaiter.run: designatedPdp is PDP=" + myPdp.getPdpId());
 553                                                 /*
 554                                                  * update function expects myPdp.isDesignated to be true.
 555                                                  */
 556                                                 try {
 557                                                         //Keep the order like this.  StateManagement is last since it triggers controller init
 558                                                         myPdp.setDesignated(true);
 559                                                         pdpsConnector.setDesignated(myPdp, true);
 560                                                         isDesignated = true;
 561                                                         String standbyStatus = stateManagement.getStandbyStatus();
 562                                                         if(!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)){
 563                                                                 /*
 564                                                                  * Only call promote if it is not already in the right state.  Don't worry about
 565                                                                  * synching the lower level topic endpoint states.  That is done by the
 566                                                                  * refreshStateAudit.
 567                                                                  * Note that we need to fetch the session list from 'mostRecentPrimary'
 568                                                                  * at this point -- soon, 'mostRecentPrimary' will be set to this host.
 569                                                                  */
 570                                                                 this.sessions = mostRecentPrimary.getSessions();
 571                                                                 stateManagement.promote();
 572                                                         }
 573                                                 } catch (StandbyStatusException e) {
 574                                                         logger.error
 575                                                         //System.out.println
 576                                                         ("ERROR: DesignatedWaiter.run: Caught StandbyStatusException attempting to promote PDP='"
 577                                                                         + myPdp.getPdpId()
 578                                                                         + "', message="
 579                                                                         + e.getMessage());
 580                                                         myPdp.setDesignated(false);
 581                                                         pdpsConnector.setDesignated(myPdp,false);
 582                                                         isDesignated = false;
 583                                                         //If you can't promote it, demote it
 584                                                         try {
 585                                                                 String standbyStatus = stateManagement.getStandbyStatus();
 586                                                                 if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 587                                                                                 standbyStatus.equals(StateManagement.COLD_STANDBY))){
 588                                                                         /*
 589                                                                          * Only call demote if it is not already in the right state.  Don't worry about
 590                                                                          * synching the lower level topic endpoint states.  That is done by the
 591                                                                          * refreshStateAudit.
 592                                                                          */
 593                                                                         stateManagement.demote();
 594                                                                 }
 595                                                         } catch (Exception e1) {
 596                                                                 logger.error
 597                                                                 //System.out.println
 598                                                                 ("ERROR: DesignatedWaiter.run: Caught StandbyStatusException attempting to promote then demote PDP='"
 599                                                                                 + myPdp.getPdpId()
 600                                                                                 + "', message="
 601                                                                                 + e1.getMessage());
 602                                                                 System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 603                                                                                 + "from stateManagement.demote()");
 604                                                                 e1.printStackTrace();
 605                                                         }
 606
 607                                                 } catch (Exception e) {
 608                                                         logger.error
 609                                                         //System.out.println
 610                                                         ("ERROR: DesignatedWaiter.run: Caught Exception attempting to promote PDP='"
 611                                                                         + myPdp.getPdpId()
 612                                                                         + "', message="
 613                                                                         + e.getMessage());
 614                                                         myPdp.setDesignated(false);
 615                                                         pdpsConnector.setDesignated(myPdp,false);
 616                                                         isDesignated = false;
 617                                                         //If you can't promote it, demote it
 618                                                         try {
 619                                                                 String standbyStatus = stateManagement.getStandbyStatus();
 620                                                                 if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 621                                                                                 standbyStatus.equals(StateManagement.COLD_STANDBY))){
 622                                                                         /*
 623                                                                          * Only call demote if it is not already in the right state.  Don't worry about
 624                                                                          * synching the lower level topic endpoint states.  That is done by the
 625                                                                          * refreshStateAudit.
 626                                                                          */
 627                                                                         stateManagement.demote();
 628                                                                 }
 629                                                         } catch (Exception e1) {
 630                                                                 logger.error
 631                                                                 //System.out.println
 632                                                                 ("ERROR: DesignatedWaiter.run: Caught StandbyStatusException attempting to promote then demote PDP='"
 633                                                                                 + myPdp.getPdpId()
 634                                                                                 + "', message="
 635                                                                                 + e1.getMessage());
 636                                                                 System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 637                                                                                 + "from stateManagement.demote()");
 638                                                                 e1.printStackTrace();
 639                                                         }
 640
 641                                                 }
 642                                                 waitTimerLastRunDate = new Date();
 643                                                 logger.info("DesignatedWaiter.run (designatedPdp.getPdpId().equals(myPdp.getPdpId())) waitTimerLastRunDate = " + waitTimerLastRunDate);
 644
 645                                                 return;
 646                                         }
 647                                         isDesignated = false;
 648
 649                                 } // end synchronized
 650
 651                                 logger.debug
 652                                 //System.out.println
 653                                 ("DesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + "; Returning, isDesignated=" + isDesignated);
 654
 655                                 Date tmpDate = new Date();
 656                                 logger.info("DesignatedWaiter.run (end of run) waitTimerLastRunDate = " + tmpDate);
 657
 658                                 waitTimerLastRunDate = tmpDate;
 659
 660                         }catch(Exception e){
 661                                 logger.error("DesignatedWaiter.run caught an unexpected exception: " + e);
 662                                 System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception");
 663                                 e.printStackTrace();
 664                         }
 665                 } // end run
 666         }
 667
 668         public ArrayList<DroolsPdp> santizeDesignatedList(ArrayList<DroolsPdp> listOfDesignated){
 669
 670                 boolean containsDesignated = false;
 671                 boolean containsHotStandby = false;
 672                 ArrayList<DroolsPdp> listForRemoval = new ArrayList<DroolsPdp>();
 673                 for(DroolsPdp pdp : listOfDesignated){
 674                         logger.debug
 675                         //System.out.println
 676                         ("DesignatedWaiter.run sanitizing: pdp = " + pdp.getPdpId()
 677                                         + " isDesignated = " + pdp.isDesignated());
 678                         if(pdp.isDesignated()){
 679                                 containsDesignated = true;
 680                         }else {
 681                                 containsHotStandby = true;
 682                                 listForRemoval.add(pdp);
 683                         }
 684                 }
 685                 if(containsDesignated && containsHotStandby){
 686                         //remove the hot standby from the list
 687                         listOfDesignated.removeAll(listForRemoval);
 688                         containsHotStandby = false;
 689                 }
 690                 return listOfDesignated;
 691         }
 692
 693         public DroolsPdp computeMostRecentPrimary(Collection<DroolsPdp> pdps, ArrayList<DroolsPdp> listOfDesignated){
 694                 boolean containsDesignated = false;
 695                 for(DroolsPdp pdp : listOfDesignated){
 696                         if(pdp.isDesignated()){
 697                                 containsDesignated = true;
 698                         }
 699                 }
 700                 DroolsPdp mostRecentPrimary = new DroolsPdpImpl(null, true, 1, new Date(0));
 701                 mostRecentPrimary.setSiteName(null);
 702                 logger.debug
 703                 //System.out.println
 704                 ("DesignatedWaiter.run listOfDesignated.size() = " + listOfDesignated.size());
 705                 if(listOfDesignated.size() <=1){
 706                         logger.debug("DesignatedWainter.run: listOfDesignated.size <=1");
 707                         //Only one or none is designated or hot standby.  Choose the latest designated date
 708                         for(DroolsPdp pdp : pdps){
 709                                 logger.debug
 710                                 //System.out.println
 711                                 ("DesignatedWaiter.run pdp = " + pdp.getPdpId()
 712                                                 + " pdp.getDesignatedDate() = " + pdp.getDesignatedDate());
 713                                 if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 714                                         mostRecentPrimary = pdp;
 715                                         logger.debug
 716                                         //System.out.println
 717                                         ("DesignatedWaiter.run mostRecentPrimary = " + mostRecentPrimary.getPdpId());
 718                                 }
 719                         }
 720                 }else if(listOfDesignated.size() == pdps.size()){
 721                         logger.debug("DesignatedWainter.run: listOfDesignated.size = pdps.size() which is " + pdps.size());
 722                         //They are all designated or all hot standby.
 723                         mostRecentPrimary = null;
 724                         for(DroolsPdp pdp : pdps){
 725                                 if(mostRecentPrimary == null){
 726                                         mostRecentPrimary = pdp;
 727                                         continue;
 728                                 }
 729                                 if(containsDesignated){ //Choose the site of the first designated date
 730                                         if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) < 0){
 731                                                 mostRecentPrimary = pdp;
 732                                                 logger.debug
 733                                                 //System.out.println
 734                                                 ("DesignatedWaiter.run mostRecentPrimary = " + mostRecentPrimary.getPdpId());
 735                                         }
 736                                 }else{ //Choose the site with the latest designated date
 737                                         if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 738                                                 mostRecentPrimary = pdp;
 739                                                 logger.debug
 740                                                 //System.out.println
 741                                                 ("DesignatedWaiter.run mostRecentPrimary = " + mostRecentPrimary.getPdpId());
 742                                         }
 743                                 }
 744                         }
 745                 }else{
 746                         logger.debug("DesignatedWainter.run: Some but not all are designated or hot standby. ");
 747                         //Some but not all are designated or hot standby.
 748                         if(containsDesignated){
 749                                 logger.debug("DesignatedWainter.run: containsDesignated = " + containsDesignated);
 750                                 /*
 751                                  * The list only contains designated.  This is a problem.  It is most likely a race
 752                                  * condition that resulted in two thinking they should be designated. Choose the
 753                                  * site with the latest designated date for the pdp not included on the designated list.
 754                                  * This should be the site that had the last designation before this race condition
 755                                  * occurred.
 756                                  */
 757                                 for(DroolsPdp pdp : pdps){
 758                                         if(listOfDesignated.contains(pdp)){
 759                                                 continue; //Don't consider this entry
 760                                         }
 761                                         if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 762                                                 mostRecentPrimary = pdp;
 763                                                 logger.debug
 764                                                 //System.out.println
 765                                                 ("DesignatedWaiter.run mostRecentPrimary = " + mostRecentPrimary.getPdpId());
 766                                         }
 767                                 }
 768                         }else{
 769                                 logger.debug("DesignatedWainter.run: containsDesignated = " + containsDesignated);
 770                                 //The list only contains hot standby. Choose the site of the latest designated date
 771                                 for(DroolsPdp pdp : pdps){
 772                                         if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 773                                                 mostRecentPrimary = pdp;
 774                                                 logger.debug
 775                                                 //System.out.println
 776                                                 ("DesignatedWaiter.run mostRecentPrimary = " + mostRecentPrimary.getPdpId());
 777                                         }
 778                                 }
 779                         }
 780                 }
 781                 return mostRecentPrimary;
 782         }
 783
 784         public DroolsPdp computeDesignatedPdp(ArrayList<DroolsPdp> listOfDesignated, DroolsPdp mostRecentPrimary){
 785                 DroolsPdp designatedPdp = null;
 786                 DroolsPdp lowestPriorityPdp = null;
 787                 if(listOfDesignated.size() > 1){
 788                         logger.debug
 789                         //System.out.println
 790                         ("DesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " listOfDesignated.size():  " + listOfDesignated.size());
 791                         DroolsPdp rejectedPdp = null;
 792                         DroolsPdp lowestPrioritySameSite = null;
 793                         DroolsPdp lowestPriorityDifferentSite = null;
 794                         for(DroolsPdp pdp : listOfDesignated){
 795                                 // We need to determine if another PDP is the lowest priority
 796                                 if(nullSafeEquals(pdp.getSiteName(),mostRecentPrimary.getSiteName())){
 797                                         if(lowestPrioritySameSite == null){
 798                                                 if(lowestPriorityDifferentSite != null){
 799                                                         rejectedPdp = lowestPriorityDifferentSite;
 800                                                 }
 801                                                 lowestPrioritySameSite = pdp;
 802                                         }else{
 803                                                 if(pdp.getPdpId().equals((lowestPrioritySameSite.getPdpId()))){
 804                                                         continue;//nothing to compare
 805                                                 }
 806                                                 if(pdp.comparePriority(lowestPrioritySameSite) <0){
 807                                                         logger.debug
 808                                                         //System.out.println
 809                                                         ("\nDesignatedWaiter.run: myPdp" + myPdp.getPdpId() + " listOfDesignated pdp ID: " + pdp.getPdpId()
 810                                                                         + " has lower priority than pdp ID: " + lowestPrioritySameSite.getPdpId());
 811
 812                                                         //we need to reject lowestPrioritySameSite
 813                                                         rejectedPdp = lowestPrioritySameSite;
 814                                                         lowestPrioritySameSite = pdp;
 815                                                 } else{
 816                                                         //we need to reject pdp and keep lowestPrioritySameSite
 817                                                         logger.debug
 818                                                         //System.out.println
 819                                                         ("\nDesignatedWaiter.run: myPdp" + myPdp.getPdpId() + " listOfDesignated pdp ID: " + pdp.getPdpId()
 820                                                                         + " has higher priority than pdp ID: " + lowestPrioritySameSite.getPdpId());
 821                                                         rejectedPdp = pdp;
 822                                                 }
 823                                         }
 824                                 } else{
 825                                         if(lowestPrioritySameSite != null){
 826                                                 //if we already have a candidate for same site, we don't want to bother with different sites
 827                                                 rejectedPdp = pdp;
 828                                         } else{
 829                                                 if(lowestPriorityDifferentSite == null){
 830                                                         lowestPriorityDifferentSite = pdp;
 831                                                         continue;
 832                                                 }
 833                                                 if(pdp.getPdpId().equals((lowestPriorityDifferentSite.getPdpId()))){
 834                                                         continue;//nothing to compare
 835                                                 }
 836                                                 if(pdp.comparePriority(lowestPriorityDifferentSite) <0){
 837                                                         logger.debug
 838                                                         //System.out.println
 839                                                         ("\nDesignatedWaiter.run: myPdp" + myPdp.getPdpId() + " listOfDesignated pdp ID: " + pdp.getPdpId()
 840                                                                         + " has lower priority than pdp ID: " + lowestPriorityDifferentSite.getPdpId());
 841
 842                                                         //we need to reject lowestPriorityDifferentSite
 843                                                         rejectedPdp = lowestPriorityDifferentSite;
 844                                                         lowestPriorityDifferentSite = pdp;
 845                                                 } else{
 846                                                         //we need to reject pdp and keep lowestPriorityDifferentSite
 847                                                         logger.debug
 848                                                         //System.out.println
 849                                                         ("\nDesignatedWaiter.run: myPdp" + myPdp.getPdpId() + " listOfDesignated pdp ID: " + pdp.getPdpId()
 850                                                                         + " has higher priority than pdp ID: " + lowestPriorityDifferentSite.getPdpId());
 851                                                         rejectedPdp = pdp;
 852                                                 }
 853                                         }
 854                                 }
 855                                 // If the rejectedPdp is myPdp, we need to stand it down and demote it.  Each pdp is responsible
 856                                 // for demoting itself
 857                                 if(rejectedPdp != null && nullSafeEquals(rejectedPdp.getPdpId(),myPdp.getPdpId())){
 858                                         logger.debug
 859                                         //System.out.println
 860                                         ("\n\nDesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " listOfDesignated myPdp ID: " + myPdp.getPdpId()
 861                                                         + " is NOT the lowest priority.  Executing stateManagement.demote()" + "\n\n");
 862                                         // We found that myPdp is on the listOfDesignated and it is not the lowest priority
 863                                         // So, we must demote it
 864                                         try {
 865                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 866                                                 myPdp.setDesignated(false);
 867                                                 pdpsConnector.setDesignated(myPdp, false);
 868                                                 isDesignated = false;
 869                                                 String standbyStatus = stateManagement.getStandbyStatus();
 870                                                 if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 871                                                                 standbyStatus.equals(StateManagement.COLD_STANDBY))){
 872                                                         /*
 873                                                          * Only call demote if it is not already in the right state.  Don't worry about
 874                                                          * synching the lower level topic endpoint states.  That is done by the
 875                                                          * refreshStateAudit.
 876                                                          */
 877                                                         stateManagement.demote();
 878                                                 }
 879                                         } catch (Exception e) {
 880                                                 myPdp.setDesignated(false);
 881                                                 pdpsConnector.setDesignated(myPdp, false);
 882                                                 isDesignated = false;
 883                                                 logger.error
 884                                                 //System.out.println
 885                                                 ("DesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " Caught Exception attempting to demote myPdp'"
 886                                                                 + myPdp.getPdpId()
 887                                                                 + "', message="
 888                                                                 + e.getMessage());
 889                                                 System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 890                                                                 + "from stateManagement.demote()");
 891                                                 e.printStackTrace();
 892                                         }
 893                                 }
 894                         } //end: for(DroolsPdp pdp : listOfDesignated)
 895                         if(lowestPrioritySameSite != null){
 896                                 lowestPriorityPdp = lowestPrioritySameSite;
 897                         } else {
 898                                 lowestPriorityPdp = lowestPriorityDifferentSite;
 899                         }
 900                         //now we have a valid value for lowestPriorityPdp
 901                         logger.debug
 902                         //System.out.println
 903                         ("\n\nDesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " listOfDesignated found the LOWEST priority pdp ID: "
 904                                         + lowestPriorityPdp.getPdpId()
 905                                         + " It is now the designatedPpd from the perspective of myPdp ID: " + myPdp + "\n\n");
 906                         designatedPdp = lowestPriorityPdp;
 907
 908                 } else if(listOfDesignated.isEmpty()){
 909                         logger.debug
 910                         //System.out.println
 911                         ("\nDesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " listOfDesignated is: EMPTY.");
 912                         designatedPdp = null;
 913                 } else{ //only one in listOfDesignated
 914                         logger.debug
 915                         //System.out.println
 916                         ("\nDesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " listOfDesignated has ONE entry. PDP ID: "
 917                                         + listOfDesignated.get(0).getPdpId());
 918                         designatedPdp = listOfDesignated.get(0);
 919                 }
 920                 return designatedPdp;
 921
 922         }
 923
 924         private class TimerUpdateClass extends TimerTask{
 925
 926                 @Override
 927                 public void run() {
 928                         try{
 929                                 logger.info("TimerUpdateClass.run: entry");
 930                                 checkWaitTimer();
 931                                 synchronized(pdpsConnectorLock){
 932
 933                                         myPdp.setUpdatedDate(new Date());
 934                                         if(myPdp.isDesignated()){
 935                                                 myPdp.setDesignatedDate(new Date());
 936                                         }
 937                                         pdpsConnector.update(myPdp);
 938
 939                                         Date tmpDate = new Date();
 940                                         logger.info("TimerUpdateClass.run: updateWorkerLastRunDate = " + tmpDate);
 941
 942                                         updateWorkerLastRunDate = tmpDate;
 943                                 }
 944                                 logger.info("TimerUpdateClass.run.exit");
 945                         }catch(Exception e){
 946                                 logger.error("TimerUpdateClass.run caught an unexpected exception: " + e);
 947                                 System.out.println(new Date() + " TimerUpdateClass.run caught an unexpected exception");
 948                                 e.printStackTrace();
 949                         }
 950                 }
 951         }
 952         @Override
 953         public void checkThreadStatus() {
 954                 checkUpdateWorkerTimer();
 955                 checkWaitTimer();
 956         }
 957
 958         private void checkUpdateWorkerTimer(){
 959                 synchronized(checkUpdateWorkerLock){
 960                         try{
 961                                 logger.debug("checkUpdateWorkerTimer: entry");
 962                                 Date now = new Date();
 963                                 long nowMs = now.getTime();
 964                                 long updateWorkerMs = updateWorkerLastRunDate.getTime();
 965                                 //give it 2 second cushion
 966                                 if((nowMs - updateWorkerMs)  > pdpCheckInterval + 2000){
 967                                         logger.error("checkUpdateWorkerTimer: nowMs - updateWorkerMs = " + (nowMs - updateWorkerMs)
 968                                                         + ", exceeds pdpCheckInterval + 2000 = " + (pdpCheckInterval + 2000) + " Will reschedule updateWorker timer");
 969
 970                                         try{
 971                                                 updateWorker.cancel();
 972                                                 // Recalculate the time because this is a synchronized section and the thread could have
 973                                                 // been blocked.
 974                                                 now = new Date();
 975                                                 nowMs = now.getTime();
 976                                                 updateWorker = new Timer();
 977                                                 // reset the updateWorkerLastRunDate
 978                                                 updateWorkerLastRunDate = new Date(nowMs + 100);
 979                                                 //execute the first time in 100 ms
 980                                                 updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
 981                                                 logger.info("checkUpdateWorkerTimer: Scheduling updateWorker timer to start in 100 ms ");
 982                                         }catch(Exception e){
 983                                                 logger.error("checkUpdateWorkerTimer: Caught unexpected Exception: " + e);
 984                                                 System.out.println(new Date() + " checkUpdateWorkerTimer caught an unexpected exception");
 985                                                 e.printStackTrace();
 986                                                 // Recalculate the time because this is a synchronized section and the thread could have
 987                                                 // been blocked.
 988                                                 now = new Date();
 989                                                 nowMs = now.getTime();
 990                                                 updateWorker = new Timer();
 991                                                 updateWorkerLastRunDate = new Date(nowMs + 100);
 992                                                 updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
 993                                                 logger.info("checkUpdateWorkerTimer: Attempting to schedule updateWorker timer in 100 ms");
 994                                         }
 995
 996                                 }
 997                                 logger.debug("checkUpdateWorkerTimer: exit");
 998                         }catch(Exception e){
 999                                 logger.error("checkUpdateWorkerTimer: caught unexpected exception: " + e);
1000                                 System.out.println(new Date() + " checkUpdateWorkerTimer - top level - caught an unexpected exception");
1001                                 e.printStackTrace();
1002                         }
1003                 }
1004         }
1005
1006         private void checkWaitTimer(){
1007                 synchronized(checkWaitTimerLock){
1008                         try{
1009                                 logger.debug("checkWaitTimer: entry");
1010                                 Date now = new Date();
1011                                 long nowMs = now.getTime();
1012                                 long waitTimerMs = waitTimerLastRunDate.getTime();
1013
1014                                 //give it 2 times leeway
1015                                 if((nowMs - waitTimerMs)  > 2*pdpUpdateInterval){
1016                                         logger.error("checkWaitTimer: nowMs - waitTimerMs = " + (nowMs - waitTimerMs)
1017                                                         + ", exceeds pdpUpdateInterval + 2000 = " + (2*pdpUpdateInterval) + " Will reschedule waitTimer timer");
1018
1019
1020                                         try{
1021                                                 // Recalculate since the thread could have been stalled on the synchronize()
1022                                                 nowMs = (new Date()).getTime();
1023                                                 // Time to the start of the next pdpUpdateInterval multiple
1024                                                 long startMs = getDWaiterStartMs();
1025                                                 waitTimer.cancel();
1026                                                 designationWaiter = new DesignationWaiter();
1027                                                 waitTimer = new Timer();
1028                                                 waitTimerLastRunDate = new Date(nowMs + startMs);
1029                                                 waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
1030                                                 logger.info("checkWaitTimer: Scheduling waitTimer timer to start in " + startMs + " ms");
1031                                         }catch(Exception e){
1032                                                 logger.error("checkWaitTimer: Caught unexpected Exception: " + e);
1033                                                 System.out.println(new Date() + " checkWaitTimer caught an unexpected exception");
1034                                                 e.printStackTrace();
1035                                                 // Recalculate since the thread could have been stalled on the synchronize()
1036                                                 nowMs = (new Date()).getTime();
1037                                                 // Time to the start of the next pdpUpdateInterval multiple
1038                                                 long startMs = getDWaiterStartMs();
1039                                                 designationWaiter = new DesignationWaiter();
1040                                                 waitTimer = new Timer();
1041                                                 waitTimerLastRunDate = new Date(nowMs + startMs);
1042                                                 waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
1043                                                 logger.info("checkWaitTimer: Scheduling waitTimer timer in " + startMs + " ms");
1044                                         }
1045
1046                                 }
1047                                 logger.debug("checkWaitTimer: exit");
1048                         }catch(Exception e){
1049                                 logger.error("checkWaitTimer: caught unexpected exception: " + e);
1050                                 System.out.println(new Date() + " checkWaitTimer caught an unexpected exception");
1051                                 e.printStackTrace();
1052                         }
1053                 }
1054         }
1055
1056         private long getDWaiterStartMs(){
1057                 Date now = new Date();
1058
1059                 // Retrieve the ms since the epoch
1060                 long nowMs = now.getTime();
1061
1062                 // Time since the end of the last pdpUpdateInterval multiple
1063                 long nowModMs = nowMs % pdpUpdateInterval;
1064
1065                 // Time to the start of the next pdpUpdateInterval multiple
1066                 long startMs = 2*pdpUpdateInterval - nowModMs;
1067
1068                 // Give the start time a minimum of a 5 second cushion
1069                 if(startMs < 5000){
1070                         // Start at the beginning  of following interval
1071                         startMs = pdpUpdateInterval + startMs;
1072                 }
1073                 return startMs;
1074         }
1075
1076         private boolean nullSafeEquals(Object one, Object two){
1077                 if(one == null && two == null){
1078                         return true;
1079                 }
1080                 if(one != null && two != null){
1081                         return one.equals(two);
1082                 }
1083                 return false;
1084         }
1085 }