gerrit.onap Code Review - policy/drools-pdp.git/blob

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * policy-persistence
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.policy.drools.persistence;
  22
  23 import java.util.ArrayList;
  24 import java.util.Collection;
  25 import java.util.Date;
  26 import java.util.LinkedList;
  27 import java.util.List;
  28 import java.util.Timer;
  29 import java.util.TimerTask;
  30
  31 import org.openecomp.policy.common.im.StandbyStatusException;
  32 import org.openecomp.policy.common.im.StateManagement;
  33 import org.openecomp.policy.drools.core.DroolsPDPIntegrityMonitor;
  34 import org.openecomp.policy.drools.core.IntegrityMonitorProperties;
  35 import org.openecomp.policy.common.logging.flexlogger.FlexLogger;
  36 import org.openecomp.policy.common.logging.flexlogger.Logger;
  37 import org.openecomp.policy.common.logging.eelf.MessageCodes;
  38
  39 public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
  40         // get an instance of logger
  41         private final static Logger  logger = FlexLogger.getLogger(DroolsPdpsElectionHandler.class);
  42         private DroolsPdpsConnector pdpsConnector;
  43         private Object pdpsConnectorLock = new Object();
  44         private Object checkUpdateWorkerLock = new Object();
  45         private Object checkWaitTimerLock = new Object();
  46         private Object designationWaiterLock = new Object();
  47
  48         /*
  49          * Must be static, so it can be referenced by JpaDroolsPdpsConnector,
  50          * without requiring a reference to the election handler instantiation.
  51          */
  52         private static DroolsPdp myPdp;
  53
  54         private DesignationWaiter designationWaiter;
  55         private Timer updateWorker;
  56         private Timer waitTimer;
  57         private Date updateWorkerLastRunDate;
  58         private Date waitTimerLastRunDate;
  59         private int pdpCheckInterval;
  60         private int pdpUpdateInterval;
  61         private volatile boolean isDesignated;
  62         DroolsPDPIntegrityMonitor droolsPdpIntegrityMonitor;
  63         StateManagement stateManagement;
  64
  65         public DroolsPdpsElectionHandler(DroolsPdpsConnector pdps, DroolsPdp myPdp, DroolsPDPIntegrityMonitor droolsPdpIntegrityMonitor){
  66                 this.pdpsConnector = pdps;
  67                 DroolsPdpsElectionHandler.myPdp = myPdp;
  68                 this.isDesignated = false;
  69                 this.droolsPdpIntegrityMonitor = droolsPdpIntegrityMonitor;
  70                 this.stateManagement = droolsPdpIntegrityMonitor.getStateManager();
  71                 pdpCheckInterval = 3000;
  72                 try{
  73                         pdpCheckInterval = Integer.parseInt(IntegrityMonitorProperties.getProperty(IntegrityMonitorProperties.PDP_CHECK_INVERVAL));
  74                 }catch(Exception e){
  75                         logger.error
  76                         //System.out.println
  77                         (MessageCodes.EXCEPTION_ERROR ,e, "Could not get pdpCheckInterval property. Using default");
  78                 }
  79                 pdpUpdateInterval = 2000;
  80                 try{
  81                         pdpUpdateInterval = Integer.parseInt(IntegrityMonitorProperties.getProperty(IntegrityMonitorProperties.PDP_UPDATE_INTERVAL));
  82                 }catch(Exception e){
  83                         logger.error
  84                         //System.out.println
  85                         (MessageCodes.EXCEPTION_ERROR, e, "Could not get pdpUpdateInterval property. Using default");
  86                 }
  87
  88                 Date now = new Date();
  89
  90                 // Retrieve the ms since the epoch
  91                 long nowMs = now.getTime();
  92
  93                 // Create the timer which will update the updateDate in DroolsPdpEntity table.
  94                 // This is the heartbeat
  95                 updateWorker = new Timer();
  96
  97                 // Schedule the heartbeat to start in 100 ms and run at pdpCheckInterval ms thereafter
  98                 updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
  99                 updateWorkerLastRunDate = new Date(nowMs + 100);
 100
 101                 // Create the timer which will run the election algorithm
 102                 waitTimer = new Timer();
 103
 104                 // Schedule it to start in startMs ms (so it will run after the updateWorker and run at pdpUpdateInterval ms thereafter
 105                 long startMs = getDWaiterStartMs();
 106                 designationWaiter = new DesignationWaiter();
 107                 waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
 108                 waitTimerLastRunDate = new Date(nowMs + startMs);
 109         }
 110
 111         public List<DroolsSessionEntity> waitForDesignation(){
 112                 while(isDesignated == false){
 113                         try {
 114                                 Thread.sleep(1000);
 115                         } catch (InterruptedException e) {
 116                                 return null;
 117                         }
 118                 }
 119                 return designationWaiter.getSessions();
 120
 121         }
 122         public List<DroolsSessionEntity> getSessions(){
 123                 return designationWaiter.getSessions();
 124         }
 125         public void updateMyPdp(){
 126                 synchronized(pdpsConnectorLock){
 127                         myPdp.setUpdatedDate(new Date());
 128                         pdpsConnector.update(myPdp);
 129                 }
 130         }
 131
 132         /*
 133          * When the JpaDroolsPdpsConnector.standDown() method is invoked, it needs
 134          * access to myPdp, so it can keep its designation status in sync with the
 135          * DB.
 136          */
 137         public static void setMyPdpDesignated(boolean designated) {
 138                 logger.debug
 139                 //System.out.println
 140                         ("setMyPdpDesignated: designated=" + designated);
 141                 myPdp.setDesignated(designated);
 142         }
 143
 144         private class DesignationWaiter extends TimerTask {
 145                 // get an instance of logger
 146                 private Logger  logger = FlexLogger.getLogger(DesignationWaiter.class);
 147                 private List<DroolsSessionEntity> sessions = null;
 148
 149                 public List<DroolsSessionEntity> getSessions(){
 150                         if(sessions != null){
 151                                 return sessions;
 152                         }
 153                         return new LinkedList<DroolsSessionEntity>();
 154                 }
 155                 public void run() {
 156                         try{
 157                                 logger.debug
 158                                 //System.out.println
 159                                 ("DesignatedWaiter.run: Entering");
 160
 161                                 // just here initially so code still works
 162                                 if (pdpsConnector == null) {
 163                                         waitTimerLastRunDate = new Date();
 164                                         logger.info("DesignatedWaiter.run (pdpsConnector==null) waitTimerLastRunDate = " + waitTimerLastRunDate);
 165
 166                                         return;
 167                                 }
 168
 169                                 synchronized (designationWaiterLock) {
 170
 171                                         logger.debug
 172                                         //System.out.println
 173                                         ("DesignatedWaiter.run: Entering synchronized block");
 174
 175                                         checkUpdateWorkerTimer();
 176
 177                                         //It is possible that multiple PDPs are designated lead.  So, we will make a list of all designated
 178                                         //PDPs and then decide which one really should be designated at the end.
 179                                         ArrayList<DroolsPdp> listOfDesignated = new ArrayList<DroolsPdp>();
 180
 181                                         Collection<DroolsPdp> pdps = pdpsConnector.getDroolsPdps();
 182                                         DroolsPdp designatedPdp = null;
 183                                         DroolsPdp lowestPriorityPdp = null;
 184
 185                                         logger.debug
 186                                         //System.out.println
 187                                         ("DesignatedWaiter.run: pdps.size="
 188                                                         + pdps.size());
 189
 190                                         //This is only true if all designated PDPs have failed
 191                                         boolean designatedPdpHasFailed = pdpsConnector.hasDesignatedPdpFailed(pdps);
 192                                         logger.debug
 193                                         //System.out.println
 194                                         ("DesignatedWaiter.run: designatedPdpHasFailed="
 195                                                         + designatedPdpHasFailed);
 196                                         for (DroolsPdp pdp : pdps) {
 197                                                 logger.debug
 198                                                 //System.out.println
 199                                                 ("DesignatedWaiter.run: evaluating pdp ID: " + pdp.getPdpId());
 200
 201                                                 /*
 202                                                  * Note: side effect of isPdpCurrent is that any stale but
 203                                                  * designated PDPs will be marked as un-designated.
 204                                                  */
 205                                                 boolean isCurrent = pdpsConnector.isPdpCurrent(pdp);
 206
 207                                                 /*
 208                                                  * We can't use stateManagement.getStandbyStatus() here, because
 209                                                  * we need the standbyStatus, not for this PDP, but for the PDP
 210                                                  * being processed by this loop iteration.
 211                                                  */
 212                                                 String standbyStatus = stateManagement.getStandbyStatus(pdp.getPdpId());
 213                                                 if(standbyStatus==null){
 214                                                         // Treat this case as a cold standby -- if we
 215                                                         // abort here, no sessions will be created in a
 216                                                         // single-node test environment.
 217                                                         standbyStatus = StateManagement.COLD_STANDBY;
 218                                                 }
 219
 220                                                 logger.debug
 221                                                 //System.out.println
 222                                                 ("DesignatedWaiter.run: PDP="
 223                                                                 + pdp.getPdpId() + ", isCurrent=" + isCurrent);
 224
 225                                                 /*
 226                                                  * There are 4 combinations of isDesignated and isCurrent.  We will examine each one in-turn
 227                                                  * and evaluate the each pdp in the list of pdps against each combination.
 228                                                  *
 229                                                  * This is the first combination of isDesignated and isCurrent
 230                                                  */
 231                                                 if (pdp.isDesignated()  &&  isCurrent) {
 232                                                         //It is current, but it could have a standbystatus=coldstandby / hotstandby
 233                                                         //If so, we need to stand it down and demote it
 234                                                         if(!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)){
 235                                                                 if(pdp.getPdpId().equals(myPdp.getPdpId())){
 236                                                                         logger.debug
 237                                                                         //System.out.println
 238                                                                         ("\n\nDesignatedWaiter.run: myPdp " + myPdp.getPdpId() + " is current and designated, "
 239                                                                                         + "butstandbystatus is not providingservice. "
 240                                                                                         + " Executing stateManagement.demote()" + "\n\n");
 241                                                                         // So, we must demote it
 242                                                                         try {
 243                                                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 244                                                                                 //This will change isDesignated and it can enter another if(combination) below
 245                                                                                 pdpsConnector.standDownPdp(pdp.getPdpId());
 246                                                                                 myPdp.setDesignated(false);
 247                                                                                 isDesignated = false;
 248                                                                                 if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 249                                                                                                 standbyStatus.equals(StateManagement.COLD_STANDBY))){
 250                                                                                         /*
 251                                                                                          * Only demote it if it appears it has not already been demoted. Don't worry
 252                                                                                          * about synching with the topic endpoint states.  That is done by the
 253                                                                                          * refreshStateAudit
 254                                                                                          */
 255                                                                                         stateManagement.demote();
 256                                                                                 }
 257                                                                                 //update the standbystatus to check in a later combination of isDesignated and isCurrent
 258                                                                                 standbyStatus=stateManagement.getStandbyStatus(pdp.getPdpId());
 259                                                                         } catch (Exception e) {
 260                                                                                 logger.error
 261                                                                                 //System.out.println
 262                                                                                 ("DesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " Caught Exception attempting to demote myPdp'"
 263                                                                                                 + myPdp.getPdpId()
 264                                                                                                 + "', message="
 265                                                                                                 + e.getMessage());
 266                                                                                 System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 267                                                                                                 + "from stateManagement.demote()");
 268                                                                                 e.printStackTrace();
 269                                                                         }
 270                                                                 }else{
 271                                                                         // Don't demote a remote PDP that is current.  It should catch itself
 272                                                                         logger.debug
 273                                                                         //System.out.println
 274                                                                         ("\n\nDesignatedWaiter.run: myPdp " + myPdp.getPdpId() + " is current and designated, "
 275                                                                                         + "but standbystatus is not providingservice. "
 276                                                                                         + " Cannot execute stateManagement.demote() since it it is not myPdp\n\n");
 277                                                                 }
 278
 279                                                         }else{
 280                                                                 // If we get here, it is ok to be on the list
 281                                                                 logger.debug
 282                                                                 //System.out.println
 283                                                                 ("DesignatedWaiter.run: PDP="
 284                                                                                 + pdp.getPdpId()
 285                                                                                 + " is designated, current and " + standbyStatus +".  Noting PDP as designated.  standbyStatus=" + standbyStatus);
 286                                                                 listOfDesignated.add(pdp);
 287                                                         }
 288
 289
 290                                                 }
 291
 292
 293                                                 /*
 294                                                  * The second combination of isDesignated and isCurrent
 295                                                  *
 296                                                  * PDP is designated but not current; it has failed.   So we stand it down (it doesn't matter what
 297                                                  * its standbyStatus is). None of these go on the list.
 298                                                  */
 299                                                 if (pdp.isDesignated()  &&  !isCurrent) {
 300                                                         logger.info
 301                                                         //System.out.println
 302                                                         ("INFO: DesignatedWaiter.run: PDP="
 303                                                                         + pdp.getPdpId()
 304                                                                         + " is currently designated but is not current; it has failed.  Standing down.  standbyStatus=" + standbyStatus);
 305
 306                                                         /*
 307                                                          * Changes designated to 0 but it is still potentially providing service
 308                                                          * Will affect isDesignated, so, it can enter an if(combination) below
 309                                                          */
 310                                                         pdpsConnector.standDownPdp(pdp.getPdpId());
 311
 312                                                         //need to change standbystatus to coldstandby
 313                                                         if (pdp.getPdpId().equals(myPdp.getPdpId())){
 314                                                                 logger.debug
 315                                                                 //System.out.println
 316                                                                 ("\n\nDesignatedWaiter.run: myPdp " + myPdp.getPdpId() + " is not Current. "
 317                                                                                 + " Executing stateManagement.disableFailed()" + "\n\n");
 318                                                                 // We found that myPdp is designated but not current
 319                                                                 // So, we must cause it to disableFail
 320                                                                 try {
 321                                                                         myPdp.setDesignated(false);
 322                                                                         //pdpsConnector.setDesignated(myPdp, false);//not needed?
 323                                                                         isDesignated = false;
 324                                                                         stateManagement.disableFailed();
 325                                                                         //stateManagement.demote();
 326                                                                 } catch (Exception e) {
 327                                                                         logger.error
 328                                                                         //System.out.println
 329                                                                         ("DesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " Caught Exception attempting to disableFail myPdp'"
 330                                                                                         + myPdp.getPdpId()
 331                                                                                         + "', message="
 332                                                                                         + e.getMessage());
 333                                                                         System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 334                                                                                         + "from stateManagement.disableFailed()");
 335                                                                         e.printStackTrace();
 336                                                                 }
 337                                                         } else { //it is a remote PDP that is failed
 338                                                                 logger.debug
 339                                                                 //System.out.println
 340                                                                 ("\n\nDesignatedWaiter.run: PDP " + pdp.getPdpId() + " is not Current. "
 341                                                                                 + " Executing stateManagement.disableFailed(otherResourceName)" + "\n\n");
 342                                                                 // We found a PDP is designated but not current
 343                                                                 // We already called standdown(pdp) which will change designated to false
 344                                                                 // Now we need to disableFail it to get its states in synch.  The standbyStatus
 345                                                                 // should equal coldstandby
 346                                                                 try {
 347                                                                         stateManagement.disableFailed(pdp.getPdpId());
 348                                                                         //stateManagement.demote(pdp.getPdpId());
 349                                                                 } catch (Exception e) {
 350                                                                         logger.error
 351                                                                         //System.out.println
 352                                                                         ("DesignatedWaiter.run: for PDP" + pdp.getPdpId()
 353                                                                                         + " Caught Exception attempting to disableFail(" + pdp.getPdpId() + ")'"
 354                                                                                         + pdp.getPdpId()
 355                                                                                         + "', message="
 356                                                                                         + e.getMessage());
 357                                                                         System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 358                                                                                         + "from stateManagement.disableFailed()");
 359                                                                         e.printStackTrace();
 360                                                                 }
 361
 362                                                         }
 363                                                         continue; //we are not going to do anything else with this pdp
 364                                                 }
 365
 366                                                 /*
 367                                                  * The third combination of isDesignated and isCurrent
 368                                                  * /*
 369                                                  * If a PDP is not currently designated but is providing service (erroneous, but recoverable) or hot standby
 370                                                  * we can add it to the list of possible designated if all the designated have failed
 371                                                  */
 372                                                 if (!pdp.isDesignated() && isCurrent){
 373                                                         if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 374                                                                         standbyStatus.equals(StateManagement.COLD_STANDBY))){
 375                                                                 logger.info("\n\nDesignatedWaiter.run: PDP " + pdp.getPdpId()
 376                                                                                 + " is NOT designated but IS current and"
 377                                                                                 + " has a standbystatus=" + standbyStatus);
 378                                                                 // Since it is current, we assume it can adjust its own state.
 379                                                                 // We will demote if it is myPdp
 380                                                                 if(pdp.getPdpId().equals(myPdp.getPdpId())){
 381                                                                         //demote it
 382                                                                         logger.info("DesignatedWaiter.run: PDP " + pdp.getPdpId() + " going to "
 383                                                                                         + "setDesignated = false and calling stateManagement.demote");
 384                                                                         try {
 385                                                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 386                                                                                 pdpsConnector.setDesignated(myPdp, false);
 387                                                                                 myPdp.setDesignated(false);
 388                                                                                 isDesignated = false;
 389                                                                                 //This is definitely not a redundant call.  It is attempting to correct a problem
 390                                                                                 stateManagement.demote();
 391                                                                                 //recheck the standbystatus
 392                                                                                 standbyStatus = stateManagement.getStandbyStatus(pdp.getPdpId());
 393                                                                         } catch (Exception e) {
 394                                                                                 logger.error
 395                                                                                 //System.out.println
 396                                                                                 ("DesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " Caught Exception attempting to demote myPdp'"
 397                                                                                                 + myPdp.getPdpId()
 398                                                                                                 + "', message="
 399                                                                                                 + e.getMessage());
 400                                                                                 System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 401                                                                                                 + "from stateManagement.demote()");
 402                                                                                 e.printStackTrace();
 403                                                                         }
 404
 405                                                                 }
 406                                                         }
 407                                                         if(standbyStatus.equals(StateManagement.HOT_STANDBY) && designatedPdpHasFailed){
 408                                                                 //add it to the list
 409                                                                 logger.info
 410                                                                 //System.out.println
 411                                                                 ("INFO: DesignatedWaiter.run: PDP=" + pdp.getPdpId()
 412                                                                                 + " is not designated but is " + standbyStatus + " and designated PDP has failed.  standbyStatus="
 413                                                                                 + standbyStatus);
 414                                                                 logger.info
 415                                                                 //System.out.println
 416                                                                 ("DesignatedWaiter.run: Designating PDP=" + pdp.getPdpId());
 417                                                                 listOfDesignated.add(pdp);
 418                                                         }
 419                                                         continue; //done with this one
 420                                                 }
 421
 422                                                 /*
 423                                                  * The fourth combination of isDesignated and isCurrent
 424                                                  *
 425                                                  * We are not going to put any of these on the list since it appears they have failed.
 426
 427                                                  *
 428                                                  */
 429                                                 if(!pdp.isDesignated() && !isCurrent) {
 430                                                         logger.info
 431                                                         //System.out.println
 432                                                         ("INFO: DesignatedWaiter.run: PDP="
 433                                                                         + pdp.getPdpId() + ", designated="
 434                                                                         + pdp.isDesignated() + ", current="
 435                                                                         + isCurrent
 436                                                                         + ", designatedPdpHasFailed="
 437                                                                         + designatedPdpHasFailed
 438                                                                         + ",  standbyStatus=" + standbyStatus);
 439                                                         if(!standbyStatus.equals(StateManagement.COLD_STANDBY)){
 440                                                                 //stand it down
 441                                                                 //disableFail it
 442                                                                 pdpsConnector.standDownPdp(pdp.getPdpId());
 443                                                                 if(pdp.getPdpId().equals(myPdp.getPdpId())){
 444                                                                         /*
 445                                                                          * I don't actually know how this condition could happen, but if it did, we would want
 446                                                                          * to declare it failed.
 447                                                                          */
 448                                                                         logger.debug
 449                                                                         //System.out.println
 450                                                                         ("\n\nDesignatedWaiter.run: myPdp " + myPdp.getPdpId() + " is !current and !designated, "
 451                                                                                         + " Executing stateManagement.disableFailed()" + "\n\n");
 452                                                                         // So, we must disableFail it
 453                                                                         try {
 454                                                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 455                                                                                 myPdp.setDesignated(false);
 456                                                                                 isDesignated = false;
 457                                                                                 stateManagement.disableFailed();
 458                                                                                 //stateManagement.demote();
 459                                                                         } catch (Exception e) {
 460                                                                                 logger.error
 461                                                                                 //System.out.println
 462                                                                                 ("DesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " Caught Exception attempting to disableFail myPdp'"
 463                                                                                                 + myPdp.getPdpId()
 464                                                                                                 + "', message="
 465                                                                                                 + e.getMessage());
 466                                                                                 System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 467                                                                                                 + "from stateManagement.disableFailed()");
 468                                                                                 e.printStackTrace();
 469                                                                         }
 470                                                                 }else{//it is remote
 471                                                                         logger.debug
 472                                                                         //System.out.println
 473                                                                         ("\n\nDesignatedWaiter.run: myPdp " + myPdp.getPdpId() + " is !current and !designated, "
 474                                                                                         + " Executing stateManagement.disableFailed(" + pdp.getPdpId() + ")" + "\n\n");
 475                                                                         // We already called standdown(pdp) which will change designated to false
 476                                                                         // Now we need to disableFail it to get its states in sync.  StandbyStatus = coldstandby
 477                                                                         try {
 478                                                                                 stateManagement.disableFailed(pdp.getPdpId());
 479                                                                                 //stateManagement.demote(pdp.getPdpId());
 480                                                                         } catch (Exception e) {
 481                                                                                 logger.error
 482                                                                                 //System.out.println
 483                                                                                 ("DesignatedWaiter.run: for PDP" + pdp.getPdpId()
 484                                                                                                 + " Caught Exception attempting to disableFail(" + pdp.getPdpId() + ")'"
 485                                                                                                 + pdp.getPdpId()
 486                                                                                                 + "', message="
 487                                                                                                 + e.getMessage());
 488                                                                                 System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 489                                                                                                 + "from stateManagement.disableFailed()");
 490                                                                                 e.printStackTrace();
 491                                                                         }
 492                                                                 }
 493                                                         }
 494                                                 }
 495
 496
 497                                         } // end pdps loop
 498
 499                                         /*
 500                                          * We have checked the four combinations of isDesignated and isCurrent.  Where appropriate,
 501                                          * we added the PDPs to the potential list of designated pdps
 502                                          *
 503                                          * We need to give priority to pdps on the same site that is currently being used
 504                                          * First, however, we must sanitize the list of designated to make sure their are
 505                                          * only designated members or non-designated members.  There should not be both in
 506                                          * the list. Because there are real time delays, it is possible that both types could
 507                                          * be on the list.
 508                                          */
 509
 510                                         listOfDesignated = santizeDesignatedList(listOfDesignated);
 511
 512                                         /*
 513                                          * We need to figure out the last pdp that was the primary so we can get the last site
 514                                          * name and the last session numbers.  We need to create a "dummy" droolspdp since
 515                                          * it will be used in later comparrisons and cannot be null.
 516                                          */
 517
 518                                         DroolsPdp mostRecentPrimary = computeMostRecentPrimary(pdps, listOfDesignated);
 519
 520
 521                                         /*
 522                                          * It is possible to get here with more than one pdp designated and providingservice. This normally
 523                                          * occurs when there is a race condition with multiple nodes coming up at the same time. If that is
 524                                          * the case we must determine which one is the one that should be designated and which one should
 525                                          * be demoted.
 526                                          *
 527                                          * It is possible to have 0, 1, 2 or more but not all, or all designated.
 528                                          *   If we have one designated and current, we chose it and are done
 529                                          *   If we have 2 or more, but not all, we must determine which one is in the same site as
 530                                          *   the previously designated pdp.
 531                                          */
 532
 533                                         designatedPdp = computeDesignatedPdp(listOfDesignated, mostRecentPrimary);
 534
 535
 536                                         if (designatedPdp == null) {
 537                                                 logger.warn
 538                                                 //System.out.println
 539                                                 ("WARNING: DesignatedWaiter.run: No viable PDP found to be Designated. designatedPdp still null.");
 540                                                 // Just to be sure the parameters are correctly set
 541                                                 myPdp.setDesignated(false);
 542                                                 pdpsConnector.setDesignated(myPdp,false);
 543                                                 isDesignated = false;
 544
 545                                                 waitTimerLastRunDate = new Date();
 546                                                 logger.info("DesignatedWaiter.run (designatedPdp == null) waitTimerLastRunDate = " + waitTimerLastRunDate);
 547
 548                                                 return;
 549
 550                                         } else if (designatedPdp.getPdpId().equals(myPdp.getPdpId())) {
 551                                                 logger.debug
 552                                                 //System.out.println
 553                                                 ("DesignatedWaiter.run: designatedPdp is PDP=" + myPdp.getPdpId());
 554                                                 /*
 555                                                  * update function expects myPdp.isDesignated to be true.
 556                                                  */
 557                                                 try {
 558                                                         //Keep the order like this.  StateManagement is last since it triggers controller init
 559                                                         myPdp.setDesignated(true);
 560                                                         pdpsConnector.setDesignated(myPdp, true);
 561                                                         isDesignated = true;
 562                                                         String standbyStatus = stateManagement.getStandbyStatus();
 563                                                         if(!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)){
 564                                                                 /*
 565                                                                  * Only call promote if it is not already in the right state.  Don't worry about
 566                                                                  * synching the lower level topic endpoint states.  That is done by the
 567                                                                  * refreshStateAudit.
 568                                                                  * Note that we need to fetch the session list from 'mostRecentPrimary'
 569                                                                  * at this point -- soon, 'mostRecentPrimary' will be set to this host.
 570                                                                  */
 571                                                                 this.sessions = mostRecentPrimary.getSessions();
 572                                                                 stateManagement.promote();
 573                                                         }
 574                                                 } catch (StandbyStatusException e) {
 575                                                         logger.error
 576                                                         //System.out.println
 577                                                         ("ERROR: DesignatedWaiter.run: Caught StandbyStatusException attempting to promote PDP='"
 578                                                                         + myPdp.getPdpId()
 579                                                                         + "', message="
 580                                                                         + e.getMessage());
 581                                                         myPdp.setDesignated(false);
 582                                                         pdpsConnector.setDesignated(myPdp,false);
 583                                                         isDesignated = false;
 584                                                         //If you can't promote it, demote it
 585                                                         try {
 586                                                                 String standbyStatus = stateManagement.getStandbyStatus();
 587                                                                 if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 588                                                                                 standbyStatus.equals(StateManagement.COLD_STANDBY))){
 589                                                                         /*
 590                                                                          * Only call demote if it is not already in the right state.  Don't worry about
 591                                                                          * synching the lower level topic endpoint states.  That is done by the
 592                                                                          * refreshStateAudit.
 593                                                                          */
 594                                                                         stateManagement.demote();
 595                                                                 }
 596                                                         } catch (Exception e1) {
 597                                                                 logger.error
 598                                                                 //System.out.println
 599                                                                 ("ERROR: DesignatedWaiter.run: Caught StandbyStatusException attempting to promote then demote PDP='"
 600                                                                                 + myPdp.getPdpId()
 601                                                                                 + "', message="
 602                                                                                 + e1.getMessage());
 603                                                                 System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 604                                                                                 + "from stateManagement.demote()");
 605                                                                 e1.printStackTrace();
 606                                                         }
 607
 608                                                 } catch (Exception e) {
 609                                                         logger.error
 610                                                         //System.out.println
 611                                                         ("ERROR: DesignatedWaiter.run: Caught Exception attempting to promote PDP='"
 612                                                                         + myPdp.getPdpId()
 613                                                                         + "', message="
 614                                                                         + e.getMessage());
 615                                                         myPdp.setDesignated(false);
 616                                                         pdpsConnector.setDesignated(myPdp,false);
 617                                                         isDesignated = false;
 618                                                         //If you can't promote it, demote it
 619                                                         try {
 620                                                                 String standbyStatus = stateManagement.getStandbyStatus();
 621                                                                 if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 622                                                                                 standbyStatus.equals(StateManagement.COLD_STANDBY))){
 623                                                                         /*
 624                                                                          * Only call demote if it is not already in the right state.  Don't worry about
 625                                                                          * synching the lower level topic endpoint states.  That is done by the
 626                                                                          * refreshStateAudit.
 627                                                                          */
 628                                                                         stateManagement.demote();
 629                                                                 }
 630                                                         } catch (Exception e1) {
 631                                                                 logger.error
 632                                                                 //System.out.println
 633                                                                 ("ERROR: DesignatedWaiter.run: Caught StandbyStatusException attempting to promote then demote PDP='"
 634                                                                                 + myPdp.getPdpId()
 635                                                                                 + "', message="
 636                                                                                 + e1.getMessage());
 637                                                                 System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 638                                                                                 + "from stateManagement.demote()");
 639                                                                 e1.printStackTrace();
 640                                                         }
 641
 642                                                 }
 643                                                 waitTimerLastRunDate = new Date();
 644                                                 logger.info("DesignatedWaiter.run (designatedPdp.getPdpId().equals(myPdp.getPdpId())) waitTimerLastRunDate = " + waitTimerLastRunDate);
 645
 646                                                 return;
 647                                         }
 648                                         isDesignated = false;
 649
 650                                 } // end synchronized
 651
 652                                 logger.debug
 653                                 //System.out.println
 654                                 ("DesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + "; Returning, isDesignated=" + isDesignated);
 655
 656                                 Date tmpDate = new Date();
 657                                 logger.info("DesignatedWaiter.run (end of run) waitTimerLastRunDate = " + tmpDate);
 658
 659                                 waitTimerLastRunDate = tmpDate;
 660
 661                         }catch(Exception e){
 662                                 logger.error("DesignatedWaiter.run caught an unexpected exception: " + e);
 663                                 System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception");
 664                                 e.printStackTrace();
 665                         }
 666                 } // end run
 667         }
 668
 669         public ArrayList<DroolsPdp> santizeDesignatedList(ArrayList<DroolsPdp> listOfDesignated){
 670
 671                 boolean containsDesignated = false;
 672                 boolean containsHotStandby = false;
 673                 ArrayList<DroolsPdp> listForRemoval = new ArrayList<DroolsPdp>();
 674                 for(DroolsPdp pdp : listOfDesignated){
 675                         logger.debug
 676                         //System.out.println
 677                         ("DesignatedWaiter.run sanitizing: pdp = " + pdp.getPdpId()
 678                                         + " isDesignated = " + pdp.isDesignated());
 679                         if(pdp.isDesignated()){
 680                                 containsDesignated = true;
 681                         }else {
 682                                 containsHotStandby = true;
 683                                 listForRemoval.add(pdp);
 684                         }
 685                 }
 686                 if(containsDesignated && containsHotStandby){
 687                         //remove the hot standby from the list
 688                         listOfDesignated.removeAll(listForRemoval);
 689                         containsHotStandby = false;
 690                 }
 691                 return listOfDesignated;
 692         }
 693
 694         public DroolsPdp computeMostRecentPrimary(Collection<DroolsPdp> pdps, ArrayList<DroolsPdp> listOfDesignated){
 695                 boolean containsDesignated = false;
 696                 for(DroolsPdp pdp : listOfDesignated){
 697                         if(pdp.isDesignated()){
 698                                 containsDesignated = true;
 699                         }
 700                 }
 701                 DroolsPdp mostRecentPrimary = new DroolsPdpImpl(null, true, 1, new Date(0));
 702                 mostRecentPrimary.setSiteName(null);
 703                 logger.debug
 704                 //System.out.println
 705                 ("DesignatedWaiter.run listOfDesignated.size() = " + listOfDesignated.size());
 706                 if(listOfDesignated.size() <=1){
 707                         logger.debug("DesignatedWainter.run: listOfDesignated.size <=1");
 708                         //Only one or none is designated or hot standby.  Choose the latest designated date
 709                         for(DroolsPdp pdp : pdps){
 710                                 logger.debug
 711                                 //System.out.println
 712                                 ("DesignatedWaiter.run pdp = " + pdp.getPdpId()
 713                                                 + " pdp.getDesignatedDate() = " + pdp.getDesignatedDate());
 714                                 if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 715                                         mostRecentPrimary = pdp;
 716                                         logger.debug
 717                                         //System.out.println
 718                                         ("DesignatedWaiter.run mostRecentPrimary = " + mostRecentPrimary.getPdpId());
 719                                 }
 720                         }
 721                 }else if(listOfDesignated.size() == pdps.size()){
 722                         logger.debug("DesignatedWainter.run: listOfDesignated.size = pdps.size() which is " + pdps.size());
 723                         //They are all designated or all hot standby.
 724                         mostRecentPrimary = null;
 725                         for(DroolsPdp pdp : pdps){
 726                                 if(mostRecentPrimary == null){
 727                                         mostRecentPrimary = pdp;
 728                                         continue;
 729                                 }
 730                                 if(containsDesignated){ //Choose the site of the first designated date
 731                                         if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) < 0){
 732                                                 mostRecentPrimary = pdp;
 733                                                 logger.debug
 734                                                 //System.out.println
 735                                                 ("DesignatedWaiter.run mostRecentPrimary = " + mostRecentPrimary.getPdpId());
 736                                         }
 737                                 }else{ //Choose the site with the latest designated date
 738                                         if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 739                                                 mostRecentPrimary = pdp;
 740                                                 logger.debug
 741                                                 //System.out.println
 742                                                 ("DesignatedWaiter.run mostRecentPrimary = " + mostRecentPrimary.getPdpId());
 743                                         }
 744                                 }
 745                         }
 746                 }else{
 747                         logger.debug("DesignatedWainter.run: Some but not all are designated or hot standby. ");
 748                         //Some but not all are designated or hot standby.
 749                         if(containsDesignated){
 750                                 logger.debug("DesignatedWainter.run: containsDesignated = " + containsDesignated);
 751                                 /*
 752                                  * The list only contains designated.  This is a problem.  It is most likely a race
 753                                  * condition that resulted in two thinking they should be designated. Choose the
 754                                  * site with the latest designated date for the pdp not included on the designated list.
 755                                  * This should be the site that had the last designation before this race condition
 756                                  * occurred.
 757                                  */
 758                                 for(DroolsPdp pdp : pdps){
 759                                         if(listOfDesignated.contains(pdp)){
 760                                                 continue; //Don't consider this entry
 761                                         }
 762                                         if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 763                                                 mostRecentPrimary = pdp;
 764                                                 logger.debug
 765                                                 //System.out.println
 766                                                 ("DesignatedWaiter.run mostRecentPrimary = " + mostRecentPrimary.getPdpId());
 767                                         }
 768                                 }
 769                         }else{
 770                                 logger.debug("DesignatedWainter.run: containsDesignated = " + containsDesignated);
 771                                 //The list only contains hot standby. Choose the site of the latest designated date
 772                                 for(DroolsPdp pdp : pdps){
 773                                         if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 774                                                 mostRecentPrimary = pdp;
 775                                                 logger.debug
 776                                                 //System.out.println
 777                                                 ("DesignatedWaiter.run mostRecentPrimary = " + mostRecentPrimary.getPdpId());
 778                                         }
 779                                 }
 780                         }
 781                 }
 782                 return mostRecentPrimary;
 783         }
 784
 785         public DroolsPdp computeDesignatedPdp(ArrayList<DroolsPdp> listOfDesignated, DroolsPdp mostRecentPrimary){
 786                 DroolsPdp designatedPdp = null;
 787                 DroolsPdp lowestPriorityPdp = null;
 788                 if(listOfDesignated.size() > 1){
 789                         logger.debug
 790                         //System.out.println
 791                         ("DesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " listOfDesignated.size():  " + listOfDesignated.size());
 792                         DroolsPdp rejectedPdp = null;
 793                         DroolsPdp lowestPrioritySameSite = null;
 794                         DroolsPdp lowestPriorityDifferentSite = null;
 795                         for(DroolsPdp pdp : listOfDesignated){
 796                                 // We need to determine if another PDP is the lowest priority
 797                                 if(nullSafeEquals(pdp.getSiteName(),mostRecentPrimary.getSiteName())){
 798                                         if(lowestPrioritySameSite == null){
 799                                                 if(lowestPriorityDifferentSite != null){
 800                                                         rejectedPdp = lowestPriorityDifferentSite;
 801                                                 }
 802                                                 lowestPrioritySameSite = pdp;
 803                                         }else{
 804                                                 if(pdp.getPdpId().equals((lowestPrioritySameSite.getPdpId()))){
 805                                                         continue;//nothing to compare
 806                                                 }
 807                                                 if(pdp.comparePriority(lowestPrioritySameSite) <0){
 808                                                         logger.debug
 809                                                         //System.out.println
 810                                                         ("\nDesignatedWaiter.run: myPdp" + myPdp.getPdpId() + " listOfDesignated pdp ID: " + pdp.getPdpId()
 811                                                                         + " has lower priority than pdp ID: " + lowestPrioritySameSite.getPdpId());
 812
 813                                                         //we need to reject lowestPrioritySameSite
 814                                                         rejectedPdp = lowestPrioritySameSite;
 815                                                         lowestPrioritySameSite = pdp;
 816                                                 } else{
 817                                                         //we need to reject pdp and keep lowestPrioritySameSite
 818                                                         logger.debug
 819                                                         //System.out.println
 820                                                         ("\nDesignatedWaiter.run: myPdp" + myPdp.getPdpId() + " listOfDesignated pdp ID: " + pdp.getPdpId()
 821                                                                         + " has higher priority than pdp ID: " + lowestPrioritySameSite.getPdpId());
 822                                                         rejectedPdp = pdp;
 823                                                 }
 824                                         }
 825                                 } else{
 826                                         if(lowestPrioritySameSite != null){
 827                                                 //if we already have a candidate for same site, we don't want to bother with different sites
 828                                                 rejectedPdp = pdp;
 829                                         } else{
 830                                                 if(lowestPriorityDifferentSite == null){
 831                                                         lowestPriorityDifferentSite = pdp;
 832                                                         continue;
 833                                                 }
 834                                                 if(pdp.getPdpId().equals((lowestPriorityDifferentSite.getPdpId()))){
 835                                                         continue;//nothing to compare
 836                                                 }
 837                                                 if(pdp.comparePriority(lowestPriorityDifferentSite) <0){
 838                                                         logger.debug
 839                                                         //System.out.println
 840                                                         ("\nDesignatedWaiter.run: myPdp" + myPdp.getPdpId() + " listOfDesignated pdp ID: " + pdp.getPdpId()
 841                                                                         + " has lower priority than pdp ID: " + lowestPriorityDifferentSite.getPdpId());
 842
 843                                                         //we need to reject lowestPriorityDifferentSite
 844                                                         rejectedPdp = lowestPriorityDifferentSite;
 845                                                         lowestPriorityDifferentSite = pdp;
 846                                                 } else{
 847                                                         //we need to reject pdp and keep lowestPriorityDifferentSite
 848                                                         logger.debug
 849                                                         //System.out.println
 850                                                         ("\nDesignatedWaiter.run: myPdp" + myPdp.getPdpId() + " listOfDesignated pdp ID: " + pdp.getPdpId()
 851                                                                         + " has higher priority than pdp ID: " + lowestPriorityDifferentSite.getPdpId());
 852                                                         rejectedPdp = pdp;
 853                                                 }
 854                                         }
 855                                 }
 856                                 // If the rejectedPdp is myPdp, we need to stand it down and demote it.  Each pdp is responsible
 857                                 // for demoting itself
 858                                 if(rejectedPdp != null && nullSafeEquals(rejectedPdp.getPdpId(),myPdp.getPdpId())){
 859                                         logger.debug
 860                                         //System.out.println
 861                                         ("\n\nDesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " listOfDesignated myPdp ID: " + myPdp.getPdpId()
 862                                                         + " is NOT the lowest priority.  Executing stateManagement.demote()" + "\n\n");
 863                                         // We found that myPdp is on the listOfDesignated and it is not the lowest priority
 864                                         // So, we must demote it
 865                                         try {
 866                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 867                                                 myPdp.setDesignated(false);
 868                                                 pdpsConnector.setDesignated(myPdp, false);
 869                                                 isDesignated = false;
 870                                                 String standbyStatus = stateManagement.getStandbyStatus();
 871                                                 if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 872                                                                 standbyStatus.equals(StateManagement.COLD_STANDBY))){
 873                                                         /*
 874                                                          * Only call demote if it is not already in the right state.  Don't worry about
 875                                                          * synching the lower level topic endpoint states.  That is done by the
 876                                                          * refreshStateAudit.
 877                                                          */
 878                                                         stateManagement.demote();
 879                                                 }
 880                                         } catch (Exception e) {
 881                                                 myPdp.setDesignated(false);
 882                                                 pdpsConnector.setDesignated(myPdp, false);
 883                                                 isDesignated = false;
 884                                                 logger.error
 885                                                 //System.out.println
 886                                                 ("DesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " Caught Exception attempting to demote myPdp'"
 887                                                                 + myPdp.getPdpId()
 888                                                                 + "', message="
 889                                                                 + e.getMessage());
 890                                                 System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 891                                                                 + "from stateManagement.demote()");
 892                                                 e.printStackTrace();
 893                                         }
 894                                 }
 895                         } //end: for(DroolsPdp pdp : listOfDesignated)
 896                         if(lowestPrioritySameSite != null){
 897                                 lowestPriorityPdp = lowestPrioritySameSite;
 898                         } else {
 899                                 lowestPriorityPdp = lowestPriorityDifferentSite;
 900                         }
 901                         //now we have a valid value for lowestPriorityPdp
 902                         logger.debug
 903                         //System.out.println
 904                         ("\n\nDesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " listOfDesignated found the LOWEST priority pdp ID: "
 905                                         + lowestPriorityPdp.getPdpId()
 906                                         + " It is now the designatedPpd from the perspective of myPdp ID: " + myPdp + "\n\n");
 907                         designatedPdp = lowestPriorityPdp;
 908
 909                 } else if(listOfDesignated.isEmpty()){
 910                         logger.debug
 911                         //System.out.println
 912                         ("\nDesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " listOfDesignated is: EMPTY.");
 913                         designatedPdp = null;
 914                 } else{ //only one in listOfDesignated
 915                         logger.debug
 916                         //System.out.println
 917                         ("\nDesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " listOfDesignated has ONE entry. PDP ID: "
 918                                         + listOfDesignated.get(0).getPdpId());
 919                         designatedPdp = listOfDesignated.get(0);
 920                 }
 921                 return designatedPdp;
 922
 923         }
 924
 925         private class TimerUpdateClass extends TimerTask{
 926
 927                 @Override
 928                 public void run() {
 929                         try{
 930                                 logger.info("TimerUpdateClass.run: entry");
 931                                 checkWaitTimer();
 932                                 synchronized(pdpsConnectorLock){
 933
 934                                         myPdp.setUpdatedDate(new Date());
 935                                         if(myPdp.isDesignated()){
 936                                                 myPdp.setDesignatedDate(new Date());
 937                                         }
 938                                         pdpsConnector.update(myPdp);
 939
 940                                         Date tmpDate = new Date();
 941                                         logger.info("TimerUpdateClass.run: updateWorkerLastRunDate = " + tmpDate);
 942
 943                                         updateWorkerLastRunDate = tmpDate;
 944                                 }
 945                                 logger.info("TimerUpdateClass.run.exit");
 946                         }catch(Exception e){
 947                                 logger.error("TimerUpdateClass.run caught an unexpected exception: " + e);
 948                                 System.out.println(new Date() + " TimerUpdateClass.run caught an unexpected exception");
 949                                 e.printStackTrace();
 950                         }
 951                 }
 952         }
 953         @Override
 954         public void checkThreadStatus() {
 955                 checkUpdateWorkerTimer();
 956                 checkWaitTimer();
 957         }
 958
 959         private void checkUpdateWorkerTimer(){
 960                 synchronized(checkUpdateWorkerLock){
 961                         try{
 962                                 logger.debug("checkUpdateWorkerTimer: entry");
 963                                 Date now = new Date();
 964                                 long nowMs = now.getTime();
 965                                 long updateWorkerMs = updateWorkerLastRunDate.getTime();
 966                                 //give it 2 second cushion
 967                                 if((nowMs - updateWorkerMs)  > pdpCheckInterval + 2000){
 968                                         logger.error("checkUpdateWorkerTimer: nowMs - updateWorkerMs = " + (nowMs - updateWorkerMs)
 969                                                         + ", exceeds pdpCheckInterval + 2000 = " + (pdpCheckInterval + 2000) + " Will reschedule updateWorker timer");
 970
 971                                         try{
 972                                                 updateWorker.cancel();
 973                                                 // Recalculate the time because this is a synchronized section and the thread could have
 974                                                 // been blocked.
 975                                                 now = new Date();
 976                                                 nowMs = now.getTime();
 977                                                 updateWorker = new Timer();
 978                                                 // reset the updateWorkerLastRunDate
 979                                                 updateWorkerLastRunDate = new Date(nowMs + 100);
 980                                                 //execute the first time in 100 ms
 981                                                 updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
 982                                                 logger.info("checkUpdateWorkerTimer: Scheduling updateWorker timer to start in 100 ms ");
 983                                         }catch(Exception e){
 984                                                 logger.error("checkUpdateWorkerTimer: Caught unexpected Exception: " + e);
 985                                                 System.out.println(new Date() + " checkUpdateWorkerTimer caught an unexpected exception");
 986                                                 e.printStackTrace();
 987                                                 // Recalculate the time because this is a synchronized section and the thread could have
 988                                                 // been blocked.
 989                                                 now = new Date();
 990                                                 nowMs = now.getTime();
 991                                                 updateWorker = new Timer();
 992                                                 updateWorkerLastRunDate = new Date(nowMs + 100);
 993                                                 updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
 994                                                 logger.info("checkUpdateWorkerTimer: Attempting to schedule updateWorker timer in 100 ms");
 995                                         }
 996
 997                                 }
 998                                 logger.debug("checkUpdateWorkerTimer: exit");
 999                         }catch(Exception e){
1000                                 logger.error("checkUpdateWorkerTimer: caught unexpected exception: " + e);
1001                                 System.out.println(new Date() + " checkUpdateWorkerTimer - top level - caught an unexpected exception");
1002                                 e.printStackTrace();
1003                         }
1004                 }
1005         }
1006
1007         private void checkWaitTimer(){
1008                 synchronized(checkWaitTimerLock){
1009                         try{
1010                                 logger.debug("checkWaitTimer: entry");
1011                                 Date now = new Date();
1012                                 long nowMs = now.getTime();
1013                                 long waitTimerMs = waitTimerLastRunDate.getTime();
1014
1015                                 //give it 2 times leeway
1016                                 if((nowMs - waitTimerMs)  > 2*pdpUpdateInterval){
1017                                         logger.error("checkWaitTimer: nowMs - waitTimerMs = " + (nowMs - waitTimerMs)
1018                                                         + ", exceeds pdpUpdateInterval + 2000 = " + (2*pdpUpdateInterval) + " Will reschedule waitTimer timer");
1019
1020
1021                                         try{
1022                                                 // Recalculate since the thread could have been stalled on the synchronize()
1023                                                 nowMs = (new Date()).getTime();
1024                                                 // Time to the start of the next pdpUpdateInterval multiple
1025                                                 long startMs = getDWaiterStartMs();
1026                                                 waitTimer.cancel();
1027                                                 designationWaiter = new DesignationWaiter();
1028                                                 waitTimer = new Timer();
1029                                                 waitTimerLastRunDate = new Date(nowMs + startMs);
1030                                                 waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
1031                                                 logger.info("checkWaitTimer: Scheduling waitTimer timer to start in " + startMs + " ms");
1032                                         }catch(Exception e){
1033                                                 logger.error("checkWaitTimer: Caught unexpected Exception: " + e);
1034                                                 System.out.println(new Date() + " checkWaitTimer caught an unexpected exception");
1035                                                 e.printStackTrace();
1036                                                 // Recalculate since the thread could have been stalled on the synchronize()
1037                                                 nowMs = (new Date()).getTime();
1038                                                 // Time to the start of the next pdpUpdateInterval multiple
1039                                                 long startMs = getDWaiterStartMs();
1040                                                 designationWaiter = new DesignationWaiter();
1041                                                 waitTimer = new Timer();
1042                                                 waitTimerLastRunDate = new Date(nowMs + startMs);
1043                                                 waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
1044                                                 logger.info("checkWaitTimer: Scheduling waitTimer timer in " + startMs + " ms");
1045                                         }
1046
1047                                 }
1048                                 logger.debug("checkWaitTimer: exit");
1049                         }catch(Exception e){
1050                                 logger.error("checkWaitTimer: caught unexpected exception: " + e);
1051                                 System.out.println(new Date() + " checkWaitTimer caught an unexpected exception");
1052                                 e.printStackTrace();
1053                         }
1054                 }
1055         }
1056
1057         private long getDWaiterStartMs(){
1058                 Date now = new Date();
1059
1060                 // Retrieve the ms since the epoch
1061                 long nowMs = now.getTime();
1062
1063                 // Time since the end of the last pdpUpdateInterval multiple
1064                 long nowModMs = nowMs % pdpUpdateInterval;
1065
1066                 // Time to the start of the next pdpUpdateInterval multiple
1067                 long startMs = 2*pdpUpdateInterval - nowModMs;
1068
1069                 // Give the start time a minimum of a 5 second cushion
1070                 if(startMs < 5000){
1071                         // Start at the beginning  of following interval
1072                         startMs = pdpUpdateInterval + startMs;
1073                 }
1074                 return startMs;
1075         }
1076
1077         private boolean nullSafeEquals(Object one, Object two){
1078                 if(one == null && two == null){
1079                         return true;
1080                 }
1081                 if(one != null && two != null){
1082                         return one.equals(two);
1083                 }
1084                 return false;
1085         }
1086 }