gerrit.onap Code Review - policy/drools-pdp.git/blob

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * policy-persistence
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.policy.drools.persistence;
  22
  23 import java.util.ArrayList;
  24 import java.util.Collection;
  25 import java.util.Date;
  26 import java.util.LinkedList;
  27 import java.util.List;
  28 import java.util.Timer;
  29 import java.util.TimerTask;
  30
  31 import org.openecomp.policy.common.im.StandbyStatusException;
  32 import org.openecomp.policy.common.im.StateManagement;
  33 import org.openecomp.policy.drools.core.DroolsPDPIntegrityMonitor;
  34 import org.openecomp.policy.drools.core.IntegrityMonitorProperties;
  35 import org.openecomp.policy.common.logging.flexlogger.FlexLogger;
  36 import org.openecomp.policy.common.logging.flexlogger.Logger;
  37 import org.openecomp.policy.common.logging.eelf.MessageCodes;
  38
  39 public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
  40         // get an instance of logger
  41         private final static Logger  logger = FlexLogger.getLogger(DroolsPdpsElectionHandler.class);
  42         private DroolsPdpsConnector pdpsConnector;
  43         private Object pdpsConnectorLock = new Object();
  44         private Object checkUpdateWorkerLock = new Object();
  45         private Object checkWaitTimerLock = new Object();
  46         private Object designationWaiterLock = new Object();
  47
  48         /*
  49          * Must be static, so it can be referenced by JpaDroolsPdpsConnector,
  50          * without requiring a reference to the election handler instantiation.
  51          */
  52         private static DroolsPdp myPdp;
  53
  54         private DesignationWaiter designationWaiter;
  55         private Timer updateWorker;
  56         private Timer waitTimer;
  57         private Date updateWorkerLastRunDate;
  58         private Date waitTimerLastRunDate;
  59         private int pdpCheckInterval;
  60         private int pdpUpdateInterval;
  61         private volatile boolean isDesignated;
  62         DroolsPDPIntegrityMonitor droolsPdpIntegrityMonitor;
  63         StateManagement stateManagement;
  64
  65         public DroolsPdpsElectionHandler(DroolsPdpsConnector pdps, DroolsPdp myPdp, DroolsPDPIntegrityMonitor droolsPdpIntegrityMonitor){
  66                 this.pdpsConnector = pdps;
  67                 DroolsPdpsElectionHandler.myPdp = myPdp;
  68                 this.isDesignated = false;
  69                 this.droolsPdpIntegrityMonitor = droolsPdpIntegrityMonitor;
  70                 this.stateManagement = droolsPdpIntegrityMonitor.getStateManager();
  71                 pdpCheckInterval = 3000;
  72                 try{
  73                         pdpCheckInterval = Integer.parseInt(IntegrityMonitorProperties.getProperty(IntegrityMonitorProperties.PDP_CHECK_INVERVAL));
  74                 }catch(Exception e){
  75                         logger.error
  76                         //System.out.println
  77                         (MessageCodes.EXCEPTION_ERROR ,e, "Could not get pdpCheckInterval property. Using default");
  78                 }
  79                 pdpUpdateInterval = 2000;
  80                 try{
  81                         pdpUpdateInterval = Integer.parseInt(IntegrityMonitorProperties.getProperty(IntegrityMonitorProperties.PDP_UPDATE_INTERVAL));
  82                 }catch(Exception e){
  83                         logger.error
  84                         //System.out.println
  85                         (MessageCodes.EXCEPTION_ERROR, e, "Could not get pdpUpdateInterval property. Using default");
  86                 }
  87
  88                 Date now = new Date();
  89
  90                 // Retrieve the ms since the epoch
  91                 long nowMs = now.getTime();
  92
  93                 // Create the timer which will update the updateDate in DroolsPdpEntity table.
  94                 // This is the heartbeat
  95                 updateWorker = new Timer();
  96
  97                 // Schedule the heartbeat to start in 100 ms and run at pdpCheckInterval ms thereafter
  98                 updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
  99                 updateWorkerLastRunDate = new Date(nowMs + 100);
 100
 101                 // Create the timer which will run the election algorithm
 102                 waitTimer = new Timer();
 103
 104                 // Schedule it to start in startMs ms (so it will run after the updateWorker and run at pdpUpdateInterval ms thereafter
 105                 long startMs = getDWaiterStartMs();
 106                 designationWaiter = new DesignationWaiter();
 107                 waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
 108                 waitTimerLastRunDate = new Date(nowMs + startMs);
 109         }
 110
 111         public List<DroolsSessionEntity> waitForDesignation(){
 112                 while(isDesignated == false){
 113                         try {
 114                                 Thread.sleep(1000);
 115                         } catch (InterruptedException e) {
 116                                 return null;
 117                         }
 118                 }
 119                 return designationWaiter.getSessions();
 120
 121         }
 122         public List<DroolsSessionEntity> getSessions(){
 123                 return designationWaiter.getSessions();
 124         }
 125         public void updateMyPdp(){
 126                 synchronized(pdpsConnectorLock){
 127                         myPdp.setUpdatedDate(new Date());
 128                         pdpsConnector.update(myPdp);
 129                 }
 130         }
 131
 132         /*
 133          * When the JpaDroolsPdpsConnector.standDown() method is invoked, it needs
 134          * access to myPdp, so it can keep its designation status in sync with the
 135          * DB.
 136          */
 137         public static void setMyPdpDesignated(boolean designated) {
 138                 logger.debug
 139                 //System.out.println
 140                         ("setMyPdpDesignated: designated=" + designated);
 141                 myPdp.setDesignated(designated);
 142         }
 143
 144         private class DesignationWaiter extends TimerTask {
 145                 // get an instance of logger
 146                 private Logger  logger = FlexLogger.getLogger(DesignationWaiter.class);
 147                 private List<DroolsSessionEntity> sessions = null;
 148
 149                 public List<DroolsSessionEntity> getSessions(){
 150                         if(sessions != null){
 151                                 return sessions;
 152                         }
 153                         return new LinkedList<DroolsSessionEntity>();
 154                 }
 155                 public void run() {
 156                         try{
 157                                 logger.debug
 158                                 //System.out.println
 159                                 ("DesignatedWaiter.run: Entering");
 160
 161                                 // just here initially so code still works
 162                                 if (pdpsConnector == null) {
 163                                         waitTimerLastRunDate = new Date();
 164                                         logger.info("DesignatedWaiter.run (pdpsConnector==null) waitTimerLastRunDate = " + waitTimerLastRunDate);
 165
 166                                         return;
 167                                 }
 168
 169                                 synchronized (designationWaiterLock) {
 170
 171                                         logger.debug
 172                                         //System.out.println
 173                                         ("DesignatedWaiter.run: Entering synchronized block");
 174
 175                                         checkUpdateWorkerTimer();
 176
 177                                         //It is possible that multiple PDPs are designated lead.  So, we will make a list of all designated
 178                                         //PDPs and then decide which one really should be designated at the end.
 179                                         ArrayList<DroolsPdp> listOfDesignated = new ArrayList<DroolsPdp>();
 180
 181                                         Collection<DroolsPdp> pdps = pdpsConnector.getDroolsPdps();
 182                                         DroolsPdp designatedPdp = null;
 183                                         DroolsPdp lowestPriorityPdp = null;
 184
 185                                         logger.debug
 186                                         //System.out.println
 187                                         ("DesignatedWaiter.run: pdps.size="
 188                                                         + pdps.size());
 189
 190                                         //This is only true if all designated PDPs have failed
 191                                         boolean designatedPdpHasFailed = pdpsConnector.hasDesignatedPdpFailed(pdps);
 192                                         logger.debug
 193                                         //System.out.println
 194                                         ("DesignatedWaiter.run: designatedPdpHasFailed="
 195                                                         + designatedPdpHasFailed);
 196                                         for (DroolsPdp pdp : pdps) {
 197                                                 logger.debug
 198                                                 //System.out.println
 199                                                 ("DesignatedWaiter.run: evaluating pdp ID: " + pdp.getPdpId());
 200
 201                                                 /*
 202                                                  * Note: side effect of isPdpCurrent is that any stale but
 203                                                  * designated PDPs will be marked as un-designated.
 204                                                  */
 205                                                 boolean isCurrent = pdpsConnector.isPdpCurrent(pdp);
 206
 207                                                 /*
 208                                                  * We can't use stateManagement.getStandbyStatus() here, because
 209                                                  * we need the standbyStatus, not for this PDP, but for the PDP
 210                                                  * being processed by this loop iteration.
 211                                                  */
 212                                                 String standbyStatus = stateManagement.getStandbyStatus(pdp.getPdpId());
 213                                                 if(standbyStatus==null){
 214                                                         // Treat this case as a cold standby -- if we
 215                                                         // abort here, no sessions will be created in a
 216                                                         // single-node test environment.
 217                                                         standbyStatus = StateManagement.COLD_STANDBY;
 218                                                 }
 219
 220                                                 logger.debug
 221                                                 //System.out.println
 222                                                 ("DesignatedWaiter.run: PDP="
 223                                                                 + pdp.getPdpId() + ", isCurrent=" + isCurrent);
 224
 225                                                 /*
 226                                                  * There are 4 combinations of isDesignated and isCurrent.  We will examine each one in-turn
 227                                                  * and evaluate the each pdp in the list of pdps against each combination.
 228                                                  *
 229                                                  * This is the first combination of isDesignated and isCurrent
 230                                                  */
 231                                                 if (pdp.isDesignated()  &&  isCurrent) {
 232                                                         //It is current, but it could have a standbystatus=coldstandby / hotstandby
 233                                                         //If so, we need to stand it down and demote it
 234                                                         if(!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)){
 235                                                                 if(pdp.getPdpId().equals(myPdp.getPdpId())){
 236                                                                         logger.debug
 237                                                                         //System.out.println
 238                                                                         ("\n\nDesignatedWaiter.run: myPdp " + myPdp.getPdpId() + " is current and designated, "
 239                                                                                         + "butstandbystatus is not providingservice. "
 240                                                                                         + " Executing stateManagement.demote()" + "\n\n");
 241                                                                         // So, we must demote it
 242                                                                         try {
 243                                                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 244                                                                                 //This will change isDesignated and it can enter another if(combination) below
 245                                                                                 pdpsConnector.standDownPdp(pdp.getPdpId());
 246                                                                                 myPdp.setDesignated(false);
 247                                                                                 isDesignated = false;
 248                                                                                 if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 249                                                                                                 standbyStatus.equals(StateManagement.COLD_STANDBY))){
 250                                                                                         /*
 251                                                                                          * Only demote it if it appears it has not already been demoted. Don't worry
 252                                                                                          * about synching with the topic endpoint states.  That is done by the
 253                                                                                          * refreshStateAudit
 254                                                                                          */
 255                                                                                         stateManagement.demote();
 256                                                                                 }
 257                                                                                 //update the standbystatus to check in a later combination of isDesignated and isCurrent
 258                                                                                 standbyStatus=stateManagement.getStandbyStatus(pdp.getPdpId());
 259                                                                         } catch (Exception e) {
 260                                                                                 logger.error
 261                                                                                 //System.out.println
 262                                                                                 ("DesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " Caught Exception attempting to demote myPdp'"
 263                                                                                                 + myPdp.getPdpId()
 264                                                                                                 + "', message="
 265                                                                                                 + e.getMessage());
 266                                                                                 System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 267                                                                                                 + "from stateManagement.demote()");
 268                                                                                 e.printStackTrace();
 269                                                                         }
 270                                                                 }else{
 271                                                                         // Don't demote a remote PDP that is current.  It should catch itself
 272                                                                         logger.debug
 273                                                                         //System.out.println
 274                                                                         ("\n\nDesignatedWaiter.run: myPdp " + myPdp.getPdpId() + " is current and designated, "
 275                                                                                         + "but standbystatus is not providingservice. "
 276                                                                                         + " Cannot execute stateManagement.demote() since it it is not myPdp\n\n");
 277                                                                 }
 278
 279                                                         }else{
 280                                                                 // If we get here, it is ok to be on the list
 281                                                                 logger.debug
 282                                                                 //System.out.println
 283                                                                 ("DesignatedWaiter.run: PDP="
 284                                                                                 + pdp.getPdpId()
 285                                                                                 + " is designated, current and " + standbyStatus +".  Noting PDP as designated.  standbyStatus=" + standbyStatus);
 286                                                                 listOfDesignated.add(pdp);
 287                                                         }
 288
 289
 290                                                 }
 291
 292
 293                                                 /*
 294                                                  * The second combination of isDesignated and isCurrent
 295                                                  *
 296                                                  * PDP is designated but not current; it has failed.   So we stand it down (it doesn't matter what
 297                                                  * its standbyStatus is). None of these go on the list.
 298                                                  */
 299                                                 if (pdp.isDesignated()  &&  !isCurrent) {
 300                                                         logger.info
 301                                                         //System.out.println
 302                                                         ("INFO: DesignatedWaiter.run: PDP="
 303                                                                         + pdp.getPdpId()
 304                                                                         + " is currently designated but is not current; it has failed.  Standing down.  standbyStatus=" + standbyStatus);
 305
 306                                                         /*
 307                                                          * Changes designated to 0 but it is still potentially providing service
 308                                                          * Will affect isDesignated, so, it can enter an if(combination) below
 309                                                          */
 310                                                         pdpsConnector.standDownPdp(pdp.getPdpId());
 311
 312                                                         //need to change standbystatus to coldstandby
 313                                                         if (pdp.getPdpId().equals(myPdp.getPdpId())){
 314                                                                 logger.debug
 315                                                                 //System.out.println
 316                                                                 ("\n\nDesignatedWaiter.run: myPdp " + myPdp.getPdpId() + " is not Current. "
 317                                                                                 + " Executing stateManagement.disableFailed()" + "\n\n");
 318                                                                 // We found that myPdp is designated but not current
 319                                                                 // So, we must cause it to disableFail
 320                                                                 try {
 321                                                                         myPdp.setDesignated(false);
 322                                                                         //pdpsConnector.setDesignated(myPdp, false);//not needed?
 323                                                                         isDesignated = false;
 324                                                                         stateManagement.disableFailed();
 325                                                                         //stateManagement.demote();
 326                                                                 } catch (Exception e) {
 327                                                                         logger.error
 328                                                                         //System.out.println
 329                                                                         ("DesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " Caught Exception attempting to disableFail myPdp'"
 330                                                                                         + myPdp.getPdpId()
 331                                                                                         + "', message="
 332                                                                                         + e.getMessage());
 333                                                                         System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 334                                                                                         + "from stateManagement.disableFailed()");
 335                                                                         e.printStackTrace();
 336                                                                 }
 337                                                         } else { //it is a remote PDP that is failed
 338                                                                 logger.debug
 339                                                                 //System.out.println
 340                                                                 ("\n\nDesignatedWaiter.run: PDP " + pdp.getPdpId() + " is not Current. "
 341                                                                                 + " Executing stateManagement.disableFailed(otherResourceName)" + "\n\n");
 342                                                                 // We found a PDP is designated but not current
 343                                                                 // We already called standdown(pdp) which will change designated to false
 344                                                                 // Now we need to disableFail it to get its states in synch.  The standbyStatus
 345                                                                 // should equal coldstandby
 346                                                                 try {
 347                                                                         stateManagement.disableFailed(pdp.getPdpId());
 348                                                                         //stateManagement.demote(pdp.getPdpId());
 349                                                                 } catch (Exception e) {
 350                                                                         logger.error
 351                                                                         //System.out.println
 352                                                                         ("DesignatedWaiter.run: for PDP" + pdp.getPdpId()
 353                                                                                         + " Caught Exception attempting to disableFail(" + pdp.getPdpId() + ")'"
 354                                                                                         + pdp.getPdpId()
 355                                                                                         + "', message="
 356                                                                                         + e.getMessage());
 357                                                                         System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 358                                                                                         + "from stateManagement.disableFailed()");
 359                                                                         e.printStackTrace();
 360                                                                 }
 361
 362                                                         }
 363                                                         continue; //we are not going to do anything else with this pdp
 364                                                 }
 365
 366                                                 /*
 367                                                  * The third combination of isDesignated and isCurrent
 368                                                  * /*
 369                                                  * If a PDP is not currently designated but is providing service (erroneous, but recoverable) or hot standby
 370                                                  * we can add it to the list of possible designated if all the designated have failed
 371                                                  */
 372                                                 if (!pdp.isDesignated() && isCurrent){
 373                                                         if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 374                                                                         standbyStatus.equals(StateManagement.COLD_STANDBY))){
 375                                                                 logger.info("\n\nDesignatedWaiter.run: PDP " + pdp.getPdpId()
 376                                                                                 + " is NOT designated but IS current and"
 377                                                                                 + " has a standbystatus=" + standbyStatus);
 378                                                                 // Since it is current, we assume it can adjust its own state.
 379                                                                 // We will demote if it is myPdp
 380                                                                 if(pdp.getPdpId().equals(myPdp.getPdpId())){
 381                                                                         //demote it
 382                                                                         logger.info("DesignatedWaiter.run: PDP " + pdp.getPdpId() + " going to "
 383                                                                                         + "setDesignated = false and calling stateManagement.demote");
 384                                                                         try {
 385                                                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 386                                                                                 pdpsConnector.setDesignated(myPdp, false);
 387                                                                                 myPdp.setDesignated(false);
 388                                                                                 isDesignated = false;
 389                                                                                 //This is definitely not a redundant call.  It is attempting to correct a problem
 390                                                                                 stateManagement.demote();
 391                                                                                 //recheck the standbystatus
 392                                                                                 standbyStatus = stateManagement.getStandbyStatus(pdp.getPdpId());
 393                                                                         } catch (Exception e) {
 394                                                                                 logger.error
 395                                                                                 //System.out.println
 396                                                                                 ("DesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " Caught Exception attempting to demote myPdp'"
 397                                                                                                 + myPdp.getPdpId()
 398                                                                                                 + "', message="
 399                                                                                                 + e.getMessage());
 400                                                                                 System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 401                                                                                                 + "from stateManagement.demote()");
 402                                                                                 e.printStackTrace();
 403                                                                         }
 404
 405                                                                 }
 406                                                         }
 407                                                         if(standbyStatus.equals(StateManagement.HOT_STANDBY) && designatedPdpHasFailed){
 408                                                                 //add it to the list
 409                                                                 logger.info
 410                                                                 //System.out.println
 411                                                                 ("INFO: DesignatedWaiter.run: PDP=" + pdp.getPdpId()
 412                                                                                 + " is not designated but is " + standbyStatus + " and designated PDP has failed.  standbyStatus="
 413                                                                                 + standbyStatus);
 414                                                                 logger.info
 415                                                                 //System.out.println
 416                                                                 ("DesignatedWaiter.run: Designating PDP=" + pdp.getPdpId());
 417                                                                 listOfDesignated.add(pdp);
 418                                                         }
 419                                                         continue; //done with this one
 420                                                 }
 421
 422                                                 /*
 423                                                  * The fourth combination of isDesignated and isCurrent
 424                                                  *
 425                                                  * We are not going to put any of these on the list since it appears they have failed.
 426
 427                                                  *
 428                                                  */
 429                                                 if(!pdp.isDesignated() && !isCurrent) {
 430                                                         logger.info
 431                                                         //System.out.println
 432                                                         ("INFO: DesignatedWaiter.run: PDP="
 433                                                                         + pdp.getPdpId() + ", designated="
 434                                                                         + pdp.isDesignated() + ", current="
 435                                                                         + isCurrent
 436                                                                         + ", designatedPdpHasFailed="
 437                                                                         + designatedPdpHasFailed
 438                                                                         + ",  standbyStatus=" + standbyStatus);
 439                                                         if(!standbyStatus.equals(StateManagement.COLD_STANDBY)){
 440                                                                 //stand it down
 441                                                                 //disableFail it
 442                                                                 pdpsConnector.standDownPdp(pdp.getPdpId());
 443                                                                 if(pdp.getPdpId().equals(myPdp.getPdpId())){
 444                                                                         /*
 445                                                                          * I don't actually know how this condition could happen, but if it did, we would want
 446                                                                          * to declare it failed.
 447                                                                          */
 448                                                                         logger.debug
 449                                                                         //System.out.println
 450                                                                         ("\n\nDesignatedWaiter.run: myPdp " + myPdp.getPdpId() + " is !current and !designated, "
 451                                                                                         + " Executing stateManagement.disableFailed()" + "\n\n");
 452                                                                         // So, we must disableFail it
 453                                                                         try {
 454                                                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 455                                                                                 myPdp.setDesignated(false);
 456                                                                                 isDesignated = false;
 457                                                                                 stateManagement.disableFailed();
 458                                                                                 //stateManagement.demote();
 459                                                                         } catch (Exception e) {
 460                                                                                 logger.error
 461                                                                                 //System.out.println
 462                                                                                 ("DesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " Caught Exception attempting to disableFail myPdp'"
 463                                                                                                 + myPdp.getPdpId()
 464                                                                                                 + "', message="
 465                                                                                                 + e.getMessage());
 466                                                                                 System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 467                                                                                                 + "from stateManagement.disableFailed()");
 468                                                                                 e.printStackTrace();
 469                                                                         }
 470                                                                 }else{//it is remote
 471                                                                         logger.debug
 472                                                                         //System.out.println
 473                                                                         ("\n\nDesignatedWaiter.run: myPdp " + myPdp.getPdpId() + " is !current and !designated, "
 474                                                                                         + " Executing stateManagement.disableFailed(" + pdp.getPdpId() + ")" + "\n\n");
 475                                                                         // We already called standdown(pdp) which will change designated to false
 476                                                                         // Now we need to disableFail it to get its states in sync.  StandbyStatus = coldstandby
 477                                                                         try {
 478                                                                                 stateManagement.disableFailed(pdp.getPdpId());
 479                                                                                 //stateManagement.demote(pdp.getPdpId());
 480                                                                         } catch (Exception e) {
 481                                                                                 logger.error
 482                                                                                 //System.out.println
 483                                                                                 ("DesignatedWaiter.run: for PDP" + pdp.getPdpId()
 484                                                                                                 + " Caught Exception attempting to disableFail(" + pdp.getPdpId() + ")'"
 485                                                                                                 + pdp.getPdpId()
 486                                                                                                 + "', message="
 487                                                                                                 + e.getMessage());
 488                                                                                 System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 489                                                                                                 + "from stateManagement.disableFailed()");
 490                                                                                 e.printStackTrace();
 491                                                                         }
 492                                                                 }
 493                                                         }
 494                                                 }
 495
 496
 497                                         } // end pdps loop
 498
 499                                         /*
 500                                          * We have checked the four combinations of isDesignated and isCurrent.  Where appropriate,
 501                                          * we added the PDPs to the potential list of designated pdps
 502                                          * Check if listOfDesignated is empty, has one entry or has multiple entries
 503                                          * If it has multiple designated PDPs, then we must determine if myPdp is on the list and if
 504                                          * it is the lowest priority.  If it is on the list and it is not the lowest
 505                                          * priority, it must be demoted.  Then, we must find the lowest priority
 506                                          * PDP so we can get the right list of sessions
 507                                          */
 508                                         //we need to give priority to pdps on the same site that is currently being used
 509
 510
 511                                         //we need to figure out the last pdp that was the primary so we can get the last site name and the last session numbers
 512                                         DroolsPdp mostRecentPrimary = new DroolsPdpImpl(null, true, 1, new Date(0));
 513                                         mostRecentPrimary.setSiteName(null);
 514                                         for(DroolsPdp pdp : pdps){
 515                                                 if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 516                                                         mostRecentPrimary = pdp;
 517                                                 }
 518                                         }
 519
 520                                         if(listOfDesignated.size() > 1){
 521                                                 logger.debug
 522                                                 //System.out.println
 523                                                 ("DesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " listOfDesignated.size():  " + listOfDesignated.size());
 524                                                 DroolsPdp rejectedPdp = null;
 525                                                 DroolsPdp lowestPrioritySameSite = null;
 526                                                 DroolsPdp lowestPriorityDifferentSite = null;
 527                                                 for(DroolsPdp pdp : listOfDesignated){
 528                                                         // We need to determine if another PDP is the lowest priority
 529                                                         if(nullSafeEquals(pdp.getSiteName(),mostRecentPrimary.getSiteName())){
 530                                                                 if(lowestPrioritySameSite == null){
 531                                                                         if(lowestPriorityDifferentSite != null){
 532                                                                                 rejectedPdp = lowestPriorityDifferentSite;
 533                                                                         }
 534                                                                         lowestPrioritySameSite = pdp;
 535                                                                 }else{
 536                                                                         if(pdp.getPdpId().equals((lowestPrioritySameSite.getPdpId()))){
 537                                                                                 continue;//nothing to compare
 538                                                                         }
 539                                                                         if(pdp.comparePriority(lowestPrioritySameSite) <0){
 540                                                                                 logger.debug
 541                                                                                 //System.out.println
 542                                                                                 ("\nDesignatedWaiter.run: myPdp" + myPdp.getPdpId() + " listOfDesignated pdp ID: " + pdp.getPdpId()
 543                                                                                                 + " has lower priority than pdp ID: " + lowestPrioritySameSite.getPdpId());
 544
 545                                                                                 //we need to reject lowestPrioritySameSite
 546                                                                                 rejectedPdp = lowestPrioritySameSite;
 547                                                                                 lowestPrioritySameSite = pdp;
 548                                                                         } else{
 549                                                                                 //we need to reject pdp and keep lowestPrioritySameSite
 550                                                                                 logger.debug
 551                                                                                 //System.out.println
 552                                                                                 ("\nDesignatedWaiter.run: myPdp" + myPdp.getPdpId() + " listOfDesignated pdp ID: " + pdp.getPdpId()
 553                                                                                                 + " has higher priority than pdp ID: " + lowestPrioritySameSite.getPdpId());
 554                                                                                 rejectedPdp = pdp;
 555                                                                         }
 556                                                                 }
 557                                                         } else{
 558                                                                 if(lowestPrioritySameSite != null){
 559                                                                         //if we already have a candidate for same site, we don't want to bother with different sites
 560                                                                         rejectedPdp = pdp;
 561                                                                 } else{
 562                                                                         if(lowestPriorityDifferentSite == null){
 563                                                                                 lowestPriorityDifferentSite = pdp;
 564                                                                                 continue;
 565                                                                         }
 566                                                                         if(pdp.getPdpId().equals((lowestPriorityDifferentSite.getPdpId()))){
 567                                                                                 continue;//nothing to compare
 568                                                                         }
 569                                                                         if(pdp.comparePriority(lowestPriorityDifferentSite) <0){
 570                                                                                 logger.debug
 571                                                                                 //System.out.println
 572                                                                                 ("\nDesignatedWaiter.run: myPdp" + myPdp.getPdpId() + " listOfDesignated pdp ID: " + pdp.getPdpId()
 573                                                                                                 + " has lower priority than pdp ID: " + lowestPriorityDifferentSite.getPdpId());
 574
 575                                                                                 //we need to reject lowestPriorityDifferentSite
 576                                                                                 rejectedPdp = lowestPriorityDifferentSite;
 577                                                                                 lowestPriorityDifferentSite = pdp;
 578                                                                         } else{
 579                                                                                 //we need to reject pdp and keep lowestPriorityDifferentSite
 580                                                                                 logger.debug
 581                                                                                 //System.out.println
 582                                                                                 ("\nDesignatedWaiter.run: myPdp" + myPdp.getPdpId() + " listOfDesignated pdp ID: " + pdp.getPdpId()
 583                                                                                                 + " has higher priority than pdp ID: " + lowestPriorityDifferentSite.getPdpId());
 584                                                                                 rejectedPdp = pdp;
 585                                                                         }
 586                                                                 }
 587                                                         }
 588                                                         // If the rejectedPdp is myPdp, we need to stand it down and demote it.  Each pdp is responsible
 589                                                         // for demoting itself
 590                                                         if(rejectedPdp != null && nullSafeEquals(rejectedPdp.getPdpId(),myPdp.getPdpId())){
 591                                                                 logger.debug
 592                                                                 //System.out.println
 593                                                                 ("\n\nDesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " listOfDesignated myPdp ID: " + myPdp.getPdpId()
 594                                                                                 + " is NOT the lowest priority.  Executing stateManagement.demote()" + "\n\n");
 595                                                                 // We found that myPdp is on the listOfDesignated and it is not the lowest priority
 596                                                                 // So, we must demote it
 597                                                                 try {
 598                                                                         //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 599                                                                         myPdp.setDesignated(false);
 600                                                                         pdpsConnector.setDesignated(myPdp, false);
 601                                                                         isDesignated = false;
 602                                                                         String standbyStatus = stateManagement.getStandbyStatus();
 603                                                                         if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 604                                                                                         standbyStatus.equals(StateManagement.COLD_STANDBY))){
 605                                                                                 /*
 606                                                                                  * Only call demote if it is not already in the right state.  Don't worry about
 607                                                                                  * synching the lower level topic endpoint states.  That is done by the
 608                                                                                  * refreshStateAudit.
 609                                                                                  */
 610                                                                                 stateManagement.demote();
 611                                                                         }
 612                                                                 } catch (Exception e) {
 613                                                                         myPdp.setDesignated(false);
 614                                                                         pdpsConnector.setDesignated(myPdp, false);
 615                                                                         isDesignated = false;
 616                                                                         logger.error
 617                                                                         //System.out.println
 618                                                                         ("DesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " Caught Exception attempting to demote myPdp'"
 619                                                                                         + myPdp.getPdpId()
 620                                                                                         + "', message="
 621                                                                                         + e.getMessage());
 622                                                                         System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 623                                                                                         + "from stateManagement.demote()");
 624                                                                         e.printStackTrace();
 625                                                                 }
 626                                                         }
 627                                                 } //end: for(DroolsPdp pdp : listOfDesignated)
 628                                                 if(lowestPrioritySameSite != null){
 629                                                         lowestPriorityPdp = lowestPrioritySameSite;
 630                                                 } else {
 631                                                         lowestPriorityPdp = lowestPriorityDifferentSite;
 632                                                 }
 633                                                 //now we have a valid value for lowestPriorityPdp
 634                                                 logger.debug
 635                                                 //System.out.println
 636                                                 ("\n\nDesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " listOfDesignated found the LOWEST priority pdp ID: "
 637                                                                 + lowestPriorityPdp.getPdpId()
 638                                                                 + " It is now the designatedPpd from the perspective of myPdp ID: " + myPdp + "\n\n");
 639                                                 designatedPdp = lowestPriorityPdp;
 640                                                 this.sessions = mostRecentPrimary.getSessions();
 641
 642                                         } else if(listOfDesignated.isEmpty()){
 643                                                 logger.debug
 644                                                 //System.out.println
 645                                                 ("\nDesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " listOfDesignated is: EMPTY.");
 646                                                 designatedPdp = null;
 647                                         } else{ //only one in listOfDesignated
 648                                                 logger.debug
 649                                                 //System.out.println
 650                                                 ("\nDesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + " listOfDesignated has ONE entry. PDP ID: "
 651                                                                 + listOfDesignated.get(0).getPdpId());
 652                                                 designatedPdp = listOfDesignated.get(0);
 653                                                 this.sessions = mostRecentPrimary.getSessions();
 654                                         }
 655
 656
 657                                         if (designatedPdp == null) {
 658                                                 logger.warn
 659                                                 //System.out.println
 660                                                 ("WARNING: DesignatedWaiter.run: No viable PDP found to be Designated. designatedPdp still null.");
 661                                                 // Just to be sure the parameters are correctly set
 662                                                 myPdp.setDesignated(false);
 663                                                 pdpsConnector.setDesignated(myPdp,false);
 664                                                 isDesignated = false;
 665
 666                                                 waitTimerLastRunDate = new Date();
 667                                                 logger.info("DesignatedWaiter.run (designatedPdp == null) waitTimerLastRunDate = " + waitTimerLastRunDate);
 668
 669                                                 return;
 670
 671                                         } else if (designatedPdp.getPdpId().equals(myPdp.getPdpId())) {
 672                                                 logger.debug
 673                                                 //System.out.println
 674                                                 ("DesignatedWaiter.run: designatedPdp is PDP=" + myPdp.getPdpId());
 675                                                 /*
 676                                                  * update function expects myPdp.isDesignated to be true.
 677                                                  */
 678                                                 try {
 679                                                         //Keep the order like this.  StateManagement is last since it triggers controller init
 680                                                         myPdp.setDesignated(true);
 681                                                         pdpsConnector.setDesignated(myPdp, true);
 682                                                         isDesignated = true;
 683                                                         String standbyStatus = stateManagement.getStandbyStatus();
 684                                                         if(!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)){
 685                                                                 /*
 686                                                                  * Only call promote if it is not already in the right state.  Don't worry about
 687                                                                  * synching the lower level topic endpoint states.  That is done by the
 688                                                                  * refreshStateAudit.
 689                                                                  */
 690                                                                 stateManagement.promote();
 691                                                         }
 692                                                 } catch (StandbyStatusException e) {
 693                                                         logger.error
 694                                                         //System.out.println
 695                                                         ("ERROR: DesignatedWaiter.run: Caught StandbyStatusException attempting to promote PDP='"
 696                                                                         + myPdp.getPdpId()
 697                                                                         + "', message="
 698                                                                         + e.getMessage());
 699                                                         myPdp.setDesignated(false);
 700                                                         pdpsConnector.setDesignated(myPdp,false);
 701                                                         isDesignated = false;
 702                                                         //If you can't promote it, demote it
 703                                                         try {
 704                                                                 String standbyStatus = stateManagement.getStandbyStatus();
 705                                                                 if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 706                                                                                 standbyStatus.equals(StateManagement.COLD_STANDBY))){
 707                                                                         /*
 708                                                                          * Only call demote if it is not already in the right state.  Don't worry about
 709                                                                          * synching the lower level topic endpoint states.  That is done by the
 710                                                                          * refreshStateAudit.
 711                                                                          */
 712                                                                         stateManagement.demote();
 713                                                                 }
 714                                                         } catch (Exception e1) {
 715                                                                 logger.error
 716                                                                 //System.out.println
 717                                                                 ("ERROR: DesignatedWaiter.run: Caught StandbyStatusException attempting to promote then demote PDP='"
 718                                                                                 + myPdp.getPdpId()
 719                                                                                 + "', message="
 720                                                                                 + e1.getMessage());
 721                                                                 System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 722                                                                                 + "from stateManagement.demote()");
 723                                                                 e1.printStackTrace();
 724                                                         }
 725
 726                                                 } catch (Exception e) {
 727                                                         logger.error
 728                                                         //System.out.println
 729                                                         ("ERROR: DesignatedWaiter.run: Caught Exception attempting to promote PDP='"
 730                                                                         + myPdp.getPdpId()
 731                                                                         + "', message="
 732                                                                         + e.getMessage());
 733                                                         myPdp.setDesignated(false);
 734                                                         pdpsConnector.setDesignated(myPdp,false);
 735                                                         isDesignated = false;
 736                                                         //If you can't promote it, demote it
 737                                                         try {
 738                                                                 String standbyStatus = stateManagement.getStandbyStatus();
 739                                                                 if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 740                                                                                 standbyStatus.equals(StateManagement.COLD_STANDBY))){
 741                                                                         /*
 742                                                                          * Only call demote if it is not already in the right state.  Don't worry about
 743                                                                          * synching the lower level topic endpoint states.  That is done by the
 744                                                                          * refreshStateAudit.
 745                                                                          */
 746                                                                         stateManagement.demote();
 747                                                                 }
 748                                                         } catch (Exception e1) {
 749                                                                 logger.error
 750                                                                 //System.out.println
 751                                                                 ("ERROR: DesignatedWaiter.run: Caught StandbyStatusException attempting to promote then demote PDP='"
 752                                                                                 + myPdp.getPdpId()
 753                                                                                 + "', message="
 754                                                                                 + e1.getMessage());
 755                                                                 System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception "
 756                                                                                 + "from stateManagement.demote()");
 757                                                                 e1.printStackTrace();
 758                                                         }
 759
 760                                                 }
 761                                                 waitTimerLastRunDate = new Date();
 762                                                 logger.info("DesignatedWaiter.run (designatedPdp.getPdpId().equals(myPdp.getPdpId())) waitTimerLastRunDate = " + waitTimerLastRunDate);
 763
 764                                                 return;
 765                                         }
 766                                         isDesignated = false;
 767
 768                                 } // end synchronized
 769
 770                                 logger.debug
 771                                 //System.out.println
 772                                 ("DesignatedWaiter.run: myPdp: " + myPdp.getPdpId() + "; Returning, isDesignated=" + isDesignated);
 773
 774                                 Date tmpDate = new Date();
 775                                 logger.info("DesignatedWaiter.run (end of run) waitTimerLastRunDate = " + tmpDate);
 776
 777                                 waitTimerLastRunDate = tmpDate;
 778
 779                         }catch(Exception e){
 780                                 logger.error("DesignatedWaiter.run caught an unexpected exception: " + e);
 781                                 System.out.println(new Date() + " DesignatedWaiter.run: caught unexpected exception");
 782                                 e.printStackTrace();
 783                         }
 784                 } // end run
 785         }
 786
 787         private class TimerUpdateClass extends TimerTask{
 788
 789                 @Override
 790                 public void run() {
 791                         try{
 792                                 logger.info("TimerUpdateClass.run: entry");
 793                                 checkWaitTimer();
 794                                 synchronized(pdpsConnectorLock){
 795
 796                                         myPdp.setUpdatedDate(new Date());
 797                                         if(myPdp.isDesignated()){
 798                                                 myPdp.setDesignatedDate(new Date());
 799                                         }
 800                                         pdpsConnector.update(myPdp);
 801
 802                                         Date tmpDate = new Date();
 803                                         logger.info("TimerUpdateClass.run: updateWorkerLastRunDate = " + tmpDate);
 804
 805                                         updateWorkerLastRunDate = tmpDate;
 806                                 }
 807                                 logger.info("TimerUpdateClass.run.exit");
 808                         }catch(Exception e){
 809                                 logger.error("TimerUpdateClass.run caught an unexpected exception: " + e);
 810                                 System.out.println(new Date() + " TimerUpdateClass.run caught an unexpected exception");
 811                                 e.printStackTrace();
 812                         }
 813                 }
 814         }
 815         @Override
 816         public void checkThreadStatus() {
 817                 checkUpdateWorkerTimer();
 818                 checkWaitTimer();
 819         }
 820
 821         private void checkUpdateWorkerTimer(){
 822                 synchronized(checkUpdateWorkerLock){
 823                         try{
 824                                 logger.debug("checkUpdateWorkerTimer: entry");
 825                                 Date now = new Date();
 826                                 long nowMs = now.getTime();
 827                                 long updateWorkerMs = updateWorkerLastRunDate.getTime();
 828                                 //give it 2 second cushion
 829                                 if((nowMs - updateWorkerMs)  > pdpCheckInterval + 2000){
 830                                         logger.error("checkUpdateWorkerTimer: nowMs - updateWorkerMs = " + (nowMs - updateWorkerMs)
 831                                                         + ", exceeds pdpCheckInterval + 2000 = " + (pdpCheckInterval + 2000) + " Will reschedule updateWorker timer");
 832
 833                                         try{
 834                                                 updateWorker.cancel();
 835                                                 // Recalculate the time because this is a synchronized section and the thread could have
 836                                                 // been blocked.
 837                                                 now = new Date();
 838                                                 nowMs = now.getTime();
 839                                                 updateWorker = new Timer();
 840                                                 // reset the updateWorkerLastRunDate
 841                                                 updateWorkerLastRunDate = new Date(nowMs + 100);
 842                                                 //execute the first time in 100 ms
 843                                                 updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
 844                                                 logger.info("checkUpdateWorkerTimer: Scheduling updateWorker timer to start in 100 ms ");
 845                                         }catch(Exception e){
 846                                                 logger.error("checkUpdateWorkerTimer: Caught unexpected Exception: " + e);
 847                                                 System.out.println(new Date() + " checkUpdateWorkerTimer caught an unexpected exception");
 848                                                 e.printStackTrace();
 849                                                 // Recalculate the time because this is a synchronized section and the thread could have
 850                                                 // been blocked.
 851                                                 now = new Date();
 852                                                 nowMs = now.getTime();
 853                                                 updateWorker = new Timer();
 854                                                 updateWorkerLastRunDate = new Date(nowMs + 100);
 855                                                 updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
 856                                                 logger.info("checkUpdateWorkerTimer: Attempting to schedule updateWorker timer in 100 ms");
 857                                         }
 858
 859                                 }
 860                                 logger.debug("checkUpdateWorkerTimer: exit");
 861                         }catch(Exception e){
 862                                 logger.error("checkUpdateWorkerTimer: caught unexpected exception: " + e);
 863                                 System.out.println(new Date() + " checkUpdateWorkerTimer - top level - caught an unexpected exception");
 864                                 e.printStackTrace();
 865                         }
 866                 }
 867         }
 868
 869         private void checkWaitTimer(){
 870                 synchronized(checkWaitTimerLock){
 871                         try{
 872                                 logger.debug("checkWaitTimer: entry");
 873                                 Date now = new Date();
 874                                 long nowMs = now.getTime();
 875                                 long waitTimerMs = waitTimerLastRunDate.getTime();
 876
 877                                 //give it 2 times leeway
 878                                 if((nowMs - waitTimerMs)  > 2*pdpUpdateInterval){
 879                                         logger.error("checkWaitTimer: nowMs - waitTimerMs = " + (nowMs - waitTimerMs)
 880                                                         + ", exceeds pdpUpdateInterval + 2000 = " + (2*pdpUpdateInterval) + " Will reschedule waitTimer timer");
 881
 882
 883                                         try{
 884                                                 // Recalculate since the thread could have been stalled on the synchronize()
 885                                                 nowMs = (new Date()).getTime();
 886                                                 // Time to the start of the next pdpUpdateInterval multiple
 887                                                 long startMs = getDWaiterStartMs();
 888                                                 waitTimer.cancel();
 889                                                 designationWaiter = new DesignationWaiter();
 890                                                 waitTimer = new Timer();
 891                                                 waitTimerLastRunDate = new Date(nowMs + startMs);
 892                                                 waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
 893                                                 logger.info("checkWaitTimer: Scheduling waitTimer timer to start in " + startMs + " ms");
 894                                         }catch(Exception e){
 895                                                 logger.error("checkWaitTimer: Caught unexpected Exception: " + e);
 896                                                 System.out.println(new Date() + " checkWaitTimer caught an unexpected exception");
 897                                                 e.printStackTrace();
 898                                                 // Recalculate since the thread could have been stalled on the synchronize()
 899                                                 nowMs = (new Date()).getTime();
 900                                                 // Time to the start of the next pdpUpdateInterval multiple
 901                                                 long startMs = getDWaiterStartMs();
 902                                                 designationWaiter = new DesignationWaiter();
 903                                                 waitTimer = new Timer();
 904                                                 waitTimerLastRunDate = new Date(nowMs + startMs);
 905                                                 waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
 906                                                 logger.info("checkWaitTimer: Scheduling waitTimer timer in " + startMs + " ms");
 907                                         }
 908
 909                                 }
 910                                 logger.debug("checkWaitTimer: exit");
 911                         }catch(Exception e){
 912                                 logger.error("checkWaitTimer: caught unexpected exception: " + e);
 913                                 System.out.println(new Date() + " checkWaitTimer caught an unexpected exception");
 914                                 e.printStackTrace();
 915                         }
 916                 }
 917         }
 918
 919         private long getDWaiterStartMs(){
 920                 Date now = new Date();
 921
 922                 // Retrieve the ms since the epoch
 923                 long nowMs = now.getTime();
 924
 925                 // Time since the end of the last pdpUpdateInterval multiple
 926                 long nowModMs = nowMs % pdpUpdateInterval;
 927
 928                 // Time to the start of the next pdpUpdateInterval multiple
 929                 long startMs = pdpUpdateInterval - nowModMs;
 930
 931                 // Give the start time a minimum of a 5 second cushion
 932                 if(startMs < 5000){
 933                         // Start at the beginning  of following interval
 934                         startMs = pdpUpdateInterval + startMs;
 935                 }
 936                 return startMs;
 937         }
 938
 939         private boolean nullSafeEquals(Object one, Object two){
 940                 if(one == null && two == null){
 941                         return true;
 942                 }
 943                 if(one != null && two != null){
 944                         return one.equals(two);
 945                 }
 946                 return false;
 947         }
 948 }