gerrit.onap Code Review - policy/drools-pdp.git/blob

   1 /*
   2  * ============LICENSE_START=======================================================
   3  * feature-active-standby-management
   4  * ================================================================================
   5  * Copyright (C) 2017-2019 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.policy.drools.activestandby;
  22
  23 import java.util.ArrayList;
  24 import java.util.Collection;
  25 import java.util.Date;
  26 import java.util.List;
  27 import java.util.Timer;
  28 import java.util.TimerTask;
  29 import org.onap.policy.common.im.MonitorTime;
  30 import org.onap.policy.common.im.StateManagement;
  31 import org.onap.policy.common.utils.time.CurrentTime;
  32 import org.onap.policy.drools.statemanagement.StateManagementFeatureApi;
  33 import org.onap.policy.drools.statemanagement.StateManagementFeatureApiConstants;
  34 import org.slf4j.Logger;
  35 import org.slf4j.LoggerFactory;
  36
  37 public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
  38     private static final String RUN_PRIMARY_MSG = "DesignatedWaiter.run mostRecentPrimary = {}";
  39
  40     // get an instance of logger
  41     private static final Logger  logger = LoggerFactory.getLogger(DroolsPdpsElectionHandler.class);
  42     private DroolsPdpsConnector pdpsConnector;
  43     private Object checkWaitTimerLock = new Object();
  44     private Object designationWaiterLock = new Object();
  45
  46     /*
  47      * Must be static, so it can be referenced by JpaDroolsPdpsConnector,
  48      * without requiring a reference to the election handler instantiation.
  49      */
  50     private static DroolsPdp myPdp;
  51
  52     private DesignationWaiter designationWaiter;
  53     private Timer updateWorker;
  54     private Timer waitTimer;
  55     private Date waitTimerLastRunDate;
  56
  57     // The interval between checks of the DesignationWaiter to be sure it is running.
  58     private int pdpCheckInterval;
  59
  60     // The interval between runs of the DesignationWaiter
  61     private int pdpUpdateInterval;
  62
  63     private volatile boolean isDesignated;
  64
  65     private String pdpdNowActive;
  66     private String pdpdLastActive;
  67
  68     /*
  69      * Start allSeemsWell with a value of null so that, on the first run
  70      * of the checkWaitTimer it will set the value in IntegrityMonitor
  71      * regardless of whether it needs to be set to true or false.
  72      */
  73     private Boolean allSeemsWell = null;
  74
  75     private StateManagementFeatureApi stateManagementFeature;
  76
  77     private final CurrentTime currentTime = MonitorTime.getInstance();
  78
  79     private static boolean isUnitTesting = false;
  80     private static boolean isStalled = false;
  81
  82     /**
  83      * Constructor.
  84      *
  85      * @param pdps connectors
  86      * @param myPdp pdp
  87      */
  88     public DroolsPdpsElectionHandler(DroolsPdpsConnector pdps, DroolsPdp myPdp) {
  89         if (pdps == null) {
  90             logger.error("DroolsPdpsElectinHandler(): pdpsConnector==null");
  91             throw new IllegalArgumentException("DroolsPdpsElectinHandler(): pdpsConnector==null");
  92         }
  93         if (myPdp == null) {
  94             logger.error("DroolsPdpsElectinHandler(): droolsPdp==null");
  95             throw new IllegalArgumentException("DroolsPdpsElectinHandler(): DroolsPdp==null");
  96         }
  97
  98         pdpdNowActive = null;
  99         pdpdLastActive = null;
 100         this.pdpsConnector = pdps;
 101         DroolsPdpsElectionHandler.myPdp = myPdp;
 102         this.isDesignated = false;
 103         pdpCheckInterval = 3000;
 104         try {
 105             pdpCheckInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(
 106                     ActiveStandbyProperties.PDP_CHECK_INVERVAL));
 107         } catch (Exception e) {
 108             logger.error("Could not get pdpCheckInterval property. Using default {}",pdpCheckInterval, e);
 109         }
 110         pdpUpdateInterval = 2000;
 111         try {
 112             pdpUpdateInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(
 113                     ActiveStandbyProperties.PDP_UPDATE_INTERVAL));
 114         } catch (Exception e) {
 115             logger.error("Could not get pdpUpdateInterval property. Using default {} ", pdpUpdateInterval, e);
 116         }
 117
 118         Date now = currentTime.getDate();
 119
 120         // Retrieve the ms since the epoch
 121         final long nowMs = now.getTime();
 122
 123         // Create the timer which will update the updateDate in DroolsPdpEntity table.
 124         // This is the heartbeat
 125         updateWorker = Factory.getInstance().makeTimer();
 126
 127         // Schedule the TimerUpdateClass to run at 100 ms and run at pdpCheckInterval ms thereafter
 128         // NOTE: The first run of the TimerUpdateClass results in myPdp being added to the
 129         // drools droolsPdpEntity table.
 130         updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
 131
 132         // Create the timer which will run the election algorithm
 133         waitTimer = Factory.getInstance().makeTimer();
 134
 135         // Schedule it to start in startMs ms
 136         // (so it will run after the updateWorker and run at pdpUpdateInterval ms thereafter
 137         long startMs = getDWaiterStartMs();
 138         designationWaiter = new DesignationWaiter();
 139         waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
 140         waitTimerLastRunDate = new Date(nowMs + startMs);
 141
 142         //Get the StateManagementFeature instance
 143
 144         for (StateManagementFeatureApi feature : StateManagementFeatureApiConstants.getImpl().getList()) {
 145             if (feature.getResourceName().equals(myPdp.getPdpId())) {
 146                 logger.debug("DroolsPdpsElectionHandler: Found StateManagementFeature"
 147                                 + " with resourceName: {}", myPdp.getPdpId());
 148                 stateManagementFeature = feature;
 149                 break;
 150             }
 151         }
 152         if (stateManagementFeature == null) {
 153             logger.error("DroolsPdpsElectionHandler failed to initialize.  "
 154                     + "Unable to get instance of StateManagementFeatureApi "
 155                     + "with resourceID: {}", myPdp.getPdpId());
 156         }
 157     }
 158
 159     public static void setIsUnitTesting(boolean val) {
 160         isUnitTesting = val;
 161     }
 162
 163     public static void setIsStalled(boolean val) {
 164         isStalled = val;
 165     }
 166
 167     /**
 168      * When the JpaDroolsPdpsConnector.standDown() method is invoked, it needs
 169      * access to myPdp, so it can keep its designation status in sync with the
 170      * DB.
 171      *
 172      * @param designated is designated value
 173      */
 174     public static void setMyPdpDesignated(boolean designated) {
 175         logger.debug("setMyPdpDesignated: designated= {}", designated);
 176         myPdp.setDesignated(designated);
 177     }
 178
 179     private class DesignationWaiter extends TimerTask {
 180         // get an instance of logger
 181         private final Logger  logger = LoggerFactory.getLogger(DesignationWaiter.class);
 182
 183         @Override
 184         public void run() {
 185             try {
 186                 logger.debug("DesignatedWaiter.run: Entering");
 187
 188                 //This is for testing the checkWaitTimer
 189                 if (isUnitTesting && isStalled) {
 190                     logger.debug("DesignatedWaiter.run: isUnitTesting = {} isStalled = {}",
 191                                     isUnitTesting, isStalled);
 192                     return;
 193                 }
 194
 195                 synchronized (designationWaiterLock) {
 196
 197                     logger.debug("DesignatedWaiter.run: Entering synchronized block");
 198
 199                     //It is possible that multiple PDPs are designated lead.  So, we will make a list of all designated
 200                     //PDPs and then decide which one really should be designated at the end.
 201                     List<DroolsPdp> listOfDesignated = new ArrayList<>();
 202
 203                     Collection<DroolsPdp> pdps = pdpsConnector.getDroolsPdps();
 204
 205                     logger.debug("DesignatedWaiter.run: pdps.size= {}", pdps.size());
 206
 207                     //This is only true if all designated PDPs have failed
 208                     allPdpsFailed(pdps, listOfDesignated);
 209
 210                     /*
 211                      * We have checked the four combinations of isDesignated and isCurrent.  Where appropriate,
 212                      * we added the PDPs to the potential list of designated pdps
 213                      *
 214                      * We need to give priority to pdps on the same site that is currently being used
 215                      * First, however, we must sanitize the list of designated to make sure their are
 216                      * only designated members or non-designated members.  There should not be both in
 217                      * the list. Because there are real time delays, it is possible that both types could
 218                      * be on the list.
 219                      */
 220
 221                     listOfDesignated = santizeDesignatedList(listOfDesignated);
 222
 223                     /*
 224                      * We need to figure out the last pdp that was the primary so we can get the last site
 225                      * name and the last session numbers.  We need to create a "dummy" droolspdp since
 226                      * it will be used in later comparisons and cannot be null.
 227                      */
 228
 229                     DroolsPdp mostRecentPrimary = computeMostRecentPrimary(pdps, listOfDesignated);
 230
 231                     if (mostRecentPrimary != null) {
 232                         pdpdLastActive = mostRecentPrimary.getPdpId();
 233                     }
 234
 235
 236                     /*
 237                      * It is possible to get here with more than one pdp designated and providing service. This normally
 238                      * occurs when there is a race condition with multiple nodes coming up at the same time. If that is
 239                      * the case we must determine which one is the one that should be designated and which one should
 240                      * be demoted.
 241                      *
 242                      * It is possible to have 0, 1, 2 or more but not all, or all designated.
 243                      *   If we have one designated and current, we chose it and are done
 244                      *   If we have 2 or more, but not all, we must determine which one is in the same site as
 245                      *   the previously designated pdp.
 246                      */
 247                     DroolsPdp designatedPdp = computeDesignatedPdp(listOfDesignated, mostRecentPrimary);
 248
 249                     if (designatedPdp == null) {
 250                         logger.warn("WARNING: DesignatedWaiter.run: No viable PDP found to be Designated. "
 251                             + "designatedPdp still null.");
 252                         designateNoPdp();
 253                         return;
 254                     }
 255
 256                     pdpdNowActive = designatedPdp.getPdpId();
 257
 258                     if (pdpdNowActive.equals(myPdp.getPdpId())) {
 259                         logger.debug("DesignatedWaiter.run: designatedPdp is PDP={}", myPdp.getPdpId());
 260                         designateMyPdp();
 261                         return;
 262                     }
 263
 264                     isDesignated = false;
 265
 266                 } // end synchronized
 267                 logger.debug("DesignatedWaiter.run: myPdp: {}; Returning, isDesignated= {}",
 268                                 isDesignated, myPdp.getPdpId());
 269
 270                 Date tmpDate = currentTime.getDate();
 271                 logger.debug("DesignatedWaiter.run (end of run) waitTimerLastRunDate = {}", tmpDate);
 272
 273                 waitTimerLastRunDate = tmpDate;
 274                 myPdp.setUpdatedDate(waitTimerLastRunDate);
 275                 pdpsConnector.update(myPdp);
 276
 277             } catch (Exception e) {
 278                 logger.error("DesignatedWaiter.run caught an unexpected exception: ", e);
 279             }
 280         } // end run
 281
 282         private void allPdpsFailed(Collection<DroolsPdp> pdps, List<DroolsPdp> listOfDesignated) {
 283             boolean designatedPdpHasFailed = pdpsConnector.hasDesignatedPdpFailed(pdps);
 284             logger.debug("DesignatedWaiter.run: designatedPdpHasFailed= {}", designatedPdpHasFailed);
 285             for (DroolsPdp pdp : pdps) {
 286                 logger.debug("DesignatedWaiter.run: evaluating pdp ID: {}", pdp.getPdpId());
 287
 288                 /*
 289                  * Note: side effect of isPdpCurrent is that any stale but
 290                  * designated PDPs will be marked as un-designated.
 291                  */
 292                 boolean isCurrent = pdpsConnector.isPdpCurrent(pdp);
 293
 294                 /*
 295                  * We can't use stateManagement.getStandbyStatus() here, because
 296                  * we need the standbyStatus, not for this PDP, but for the PDP
 297                  * being processed by this loop iteration.
 298                  */
 299                 String standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 300                 if (standbyStatus == null) {
 301                     // Treat this case as a cold standby -- if we
 302                     // abort here, no sessions will be created in a
 303                     // single-node test environment.
 304                     standbyStatus = StateManagement.COLD_STANDBY;
 305                 }
 306                 logger.debug("DesignatedWaiter.run: PDP= {},  isCurrent= {}", pdp.getPdpId(), isCurrent);
 307
 308                 adjustPdp(pdp, isCurrent, designatedPdpHasFailed, standbyStatus, listOfDesignated);
 309
 310
 311             } // end pdps loop
 312         }
 313
 314         private void adjustPdp(DroolsPdp pdp, boolean isCurrent, boolean designatedPdpHasFailed, String standbyStatus,
 315                         List<DroolsPdp> listOfDesignated) {
 316             /*
 317              * There are 4 combinations of isDesignated and isCurrent.  We will examine each one in-turn
 318              * and evaluate the each pdp in the list of pdps against each combination.
 319              */
 320             if (pdp.isDesignated()) {
 321                 /*
 322                  * This is the first combination of isDesignated and isCurrent
 323                  */
 324                 if (isCurrent) {
 325                     pdpDesignatedCurrent(pdp, standbyStatus, listOfDesignated);
 326
 327                 /*
 328                  * The second combination of isDesignated and isCurrent
 329                  *
 330                  * PDP is designated but not current; it has failed.
 331                  * So we stand it down (it doesn't matter what
 332                  * its standbyStatus is). None of these go on the list.
 333                  */
 334                 } else {
 335                     logger.debug("INFO: DesignatedWaiter.run: PDP= {} is currently "
 336                                     + "designated but is not current; "
 337                                     + "it has failed.  Standing down.  standbyStatus= {}",
 338                                     pdp.getPdpId(), standbyStatus);
 339                     pdpDesignatedNotCurrent(pdp);
 340                 }
 341
 342             } else {
 343                 // NOT designated
 344
 345
 346                 /*
 347                  * The third combination of isDesignated and isCurrent
 348                  * /*
 349                  * If a PDP is not currently designated but is providing service
 350                  * (erroneous, but recoverable) or hot standby
 351                  * we can add it to the list of possible designated if all the designated have failed
 352                  */
 353                 if (isCurrent) {
 354                     pdpNotDesignatedCurrent(pdp, designatedPdpHasFailed, standbyStatus,
 355                                     listOfDesignated);
 356
 357                 /*
 358                  * The fourth combination of isDesignated and isCurrent
 359                  *
 360                  * We are not going to put any of these on the list since it appears they have failed.
 361                  *
 362                  */
 363                 } else {
 364                     logger.debug("INFO: DesignatedWaiter.run: PDP= {} "
 365                                     + "designated= {}, current= {}, "
 366                                     + "designatedPdpHasFailed= {}, "
 367                                     + "standbyStatus= {}",pdp.getPdpId(),
 368                                     pdp.isDesignated(), false, designatedPdpHasFailed, standbyStatus);
 369                     pdpNotDesignatedNotCurrent(pdp, standbyStatus);
 370                 }
 371             }
 372         }
 373
 374         private void pdpDesignatedCurrent(DroolsPdp pdp, String standbyStatus, List<DroolsPdp> listOfDesignated) {
 375             //It is current, but it could have a standbystatus=coldstandby / hotstandby
 376             //If so, we need to stand it down and demote it
 377             if (!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)) {
 378                 if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 379                     logger.debug("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 380                                     + "butstandbystatus is not providingservice. "
 381                                     + " Executing stateManagement.demote()" + "\n\n", myPdp.getPdpId());
 382                     // So, we must demote it
 383                     try {
 384                         demoteMyPdp(pdp, standbyStatus);
 385                     } catch (Exception e) {
 386                         logger.error("DesignatedWaiter.run: myPdp: {} "
 387                                 + "Caught Exception attempting to demote myPdp,"
 388                                 + "message= {}", myPdp.getPdpId(), e);
 389                     }
 390                 } else {
 391                     // Don't demote a remote PDP that is current.  It should catch itself
 392                     logger.debug("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 393                                     + "but standbystatus is not providingservice. "
 394                                     + " Cannot execute stateManagement.demote() "
 395                                     + "since it it is not myPdp\n\n",
 396                                     myPdp.getPdpId());
 397                 }
 398
 399             } else {
 400                 // If we get here, it is ok to be on the list
 401                 logger.debug("DesignatedWaiter.run: PDP= {} is designated, "
 402                                 + "current and {} Noting PDP as "
 403                                 + "designated, standbyStatus= {}",
 404                                 pdp.getPdpId(), standbyStatus, standbyStatus);
 405                 listOfDesignated.add(pdp);
 406             }
 407         }
 408
 409         private void demoteMyPdp(DroolsPdp pdp, String standbyStatus) throws Exception {
 410             //Keep the order like this.  StateManagement is last since it
 411             //triggers controller shutdown
 412             //This will change isDesignated and it can enter another if(combination) below
 413             pdpsConnector.standDownPdp(pdp.getPdpId());
 414             myPdp.setDesignated(false);
 415             isDesignated = false;
 416             if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 417                     || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 418                 /*
 419                  * Only demote it if it appears it has not already been demoted. Don't worry
 420                  * about synching with the topic endpoint states.  That is done by the
 421                  * refreshStateAudit
 422                  */
 423                 stateManagementFeature.demote();
 424             }
 425         }
 426
 427         private void pdpDesignatedNotCurrent(DroolsPdp pdp) {
 428             /*
 429              * Changes designated to 0 but it is still potentially providing service
 430              * Will affect isDesignated, so, it can enter an if(combination) below
 431              */
 432             pdpsConnector.standDownPdp(pdp.getPdpId());
 433
 434             //need to change standbystatus to coldstandby
 435             if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 436                 logger.debug("\n\nDesignatedWaiter.run: myPdp {} is not Current. "
 437                                 + " Executing stateManagement.disableFailed()\n\n", myPdp.getPdpId());
 438                 // We found that myPdp is designated but not current
 439                 // So, we must cause it to disableFail
 440                 try {
 441                     myPdp.setDesignated(false);
 442                     pdpsConnector.setDesignated(myPdp, false);
 443                     isDesignated = false;
 444                     stateManagementFeature.disableFailed();
 445                 } catch (Exception e) {
 446                     logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception "
 447                             + "attempting to disableFail myPdp {}, message= {}",
 448                             myPdp.getPdpId(), myPdp.getPdpId(), e);
 449                 }
 450             } else { //it is a remote PDP that is failed
 451                 logger.debug("\n\nDesignatedWaiter.run: PDP {} is not Current. "
 452                                 + " Executing stateManagement.disableFailed(otherResourceName)\n\n",
 453                                 pdp.getPdpId() );
 454                 // We found a PDP is designated but not current
 455                 // We already called standdown(pdp) which will change designated to false
 456                 // Now we need to disableFail it to get its states in synch.  The standbyStatus
 457                 // should equal coldstandby
 458                 try {
 459                     stateManagementFeature.disableFailed(pdp.getPdpId());
 460                 } catch (Exception e) {
 461                     logger.error("DesignatedWaiter.run: for PDP {}  Caught Exception attempting to "
 462                             + "disableFail({}), message= {}",
 463                             pdp.getPdpId(), pdp.getPdpId(), e);
 464                 }
 465
 466             }
 467         }
 468
 469         private void pdpNotDesignatedCurrent(DroolsPdp pdp, boolean designatedPdpHasFailed, String standbyStatus,
 470                         List<DroolsPdp> listOfDesignated) {
 471             if (!(StateManagement.HOT_STANDBY.equals(standbyStatus)
 472                     || StateManagement.COLD_STANDBY.equals(standbyStatus))) {
 473                 logger.debug("\n\nDesignatedWaiter.run: PDP {}"
 474                                 + " is NOT designated but IS current and"
 475                                 + " has a standbystatus= {}", pdp.getPdpId(), standbyStatus);
 476                 // Since it is current, we assume it can adjust its own state.
 477                 // We will demote if it is myPdp
 478                 if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 479                     //demote it
 480                     logger.debug("DesignatedWaiter.run: PDP {} going to "
 481                                     + "setDesignated = false and calling stateManagement.demote",
 482                                     pdp.getPdpId());
 483                     try {
 484                         //Keep the order like this.
 485                         //StateManagement is last since it triggers controller shutdown
 486                         pdpsConnector.setDesignated(myPdp, false);
 487                         myPdp.setDesignated(false);
 488                         isDesignated = false;
 489                         //This is definitely not a redundant call.
 490                         //It is attempting to correct a problem
 491                         stateManagementFeature.demote();
 492                         //recheck the standbystatus
 493                         standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 494                     } catch (Exception e) {
 495                         logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception "
 496                                 + "attempting to demote myPdp {}, message = {}",  myPdp.getPdpId(),
 497                                 myPdp.getPdpId(), e);
 498                     }
 499
 500                 }
 501             }
 502             if (StateManagement.HOT_STANDBY.equals(standbyStatus) && designatedPdpHasFailed) {
 503                 //add it to the list
 504                 logger.debug("INFO: DesignatedWaiter.run: PDP= {}"
 505                                 + " is not designated but is {} and designated PDP "
 506                                 + "has failed.  standbyStatus= {}", pdp.getPdpId(),
 507                                 standbyStatus, standbyStatus);
 508                 listOfDesignated.add(pdp);
 509             }
 510         }
 511
 512         private void pdpNotDesignatedNotCurrent(DroolsPdp pdp, String standbyStatus) {
 513             if (StateManagement.COLD_STANDBY.equals(standbyStatus)) {
 514                 return;
 515             }
 516
 517             //stand it down
 518             //disableFail it
 519             pdpsConnector.standDownPdp(pdp.getPdpId());
 520             if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 521                 /*
 522                  * I don't actually know how this condition could
 523                  * happen, but if it did, we would want to declare it
 524                  * failed.
 525                  */
 526                 logger.debug("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 527                                 + " Executing stateManagement.disableFailed()\n\n",
 528                                 myPdp.getPdpId());
 529                 // So, we must disableFail it
 530                 try {
 531                     //Keep the order like this.
 532                     //StateManagement is last since it triggers controller shutdown
 533                     pdpsConnector.setDesignated(myPdp, false);
 534                     myPdp.setDesignated(false);
 535                     isDesignated = false;
 536                     stateManagementFeature.disableFailed();
 537                 } catch (Exception e) {
 538                     logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 539                             + "disableFail myPdp {}, message= {}",
 540                             myPdp.getPdpId(), myPdp.getPdpId(), e);
 541                 }
 542             } else { //it is remote
 543                 logger.debug("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 544                                 + " Executing stateManagement.disableFailed({})\n\n",
 545                                 myPdp.getPdpId(), pdp.getPdpId());
 546                 // We already called standdown(pdp) which will change designated to false
 547                 // Now we need to disableFail it to get its states in sync.
 548                 // StandbyStatus = coldstandby
 549                 try {
 550                     stateManagementFeature.disableFailed(pdp.getPdpId());
 551                 } catch (Exception e) {
 552                     logger.error("DesignatedWaiter.run: for PDP {}"
 553                             + " Caught Exception attempting to disableFail({})"
 554                             + ", message=", pdp.getPdpId(), pdp.getPdpId(), e);
 555                 }
 556             }
 557         }
 558
 559         private void designateNoPdp() {
 560             // Just to be sure the parameters are correctly set
 561             myPdp.setDesignated(false);
 562             pdpsConnector.setDesignated(myPdp,false);
 563             isDesignated = false;
 564
 565             waitTimerLastRunDate = currentTime.getDate();
 566             logger.debug("DesignatedWaiter.run (designatedPdp == null) waitTimerLastRunDate = {}",
 567                             waitTimerLastRunDate);
 568             myPdp.setUpdatedDate(waitTimerLastRunDate);
 569             pdpsConnector.update(myPdp);
 570         }
 571
 572         private void designateMyPdp() {
 573             /*
 574              * update function expects myPdp.isDesignated to be true.
 575              */
 576             try {
 577                 //Keep the order like this.  StateManagement is last since it triggers controller init
 578                 myPdp.setDesignated(true);
 579                 myPdp.setDesignatedDate(currentTime.getDate());
 580                 pdpsConnector.setDesignated(myPdp, true);
 581                 isDesignated = true;
 582                 String standbyStatus = stateManagementFeature.getStandbyStatus();
 583                 if (!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)) {
 584                     /*
 585                      * Only call promote if it is not already in the right state.  Don't worry about
 586                      * synching the lower level topic endpoint states.  That is done by the
 587                      * refreshStateAudit.
 588                      * Note that we need to fetch the session list from 'mostRecentPrimary'
 589                      * at this point -- soon, 'mostRecentPrimary' will be set to this host.
 590                      */
 591                     //this.sessions = mostRecentPrimary.getSessions();
 592                     stateManagementFeature.promote();
 593                 }
 594             } catch (Exception e) {
 595                 logger.error("ERROR: DesignatedWaiter.run: Caught Exception attempting to promote PDP={}"
 596                         + ", message=", myPdp.getPdpId(), e);
 597                 myPdp.setDesignated(false);
 598                 pdpsConnector.setDesignated(myPdp,false);
 599                 isDesignated = false;
 600                 //If you can't promote it, demote it
 601                 try {
 602                     String standbyStatus = stateManagementFeature.getStandbyStatus();
 603                     if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 604                             || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 605                         /*
 606                          * Only call demote if it is not already in the right state.  Don't worry about
 607                          * synching the lower level topic endpoint states.  That is done by the
 608                          * refreshStateAudit.
 609                          */
 610                         stateManagementFeature.demote();
 611                     }
 612                 } catch (Exception e1) {
 613                     logger.error("ERROR: DesignatedWaiter.run: Caught StandbyStatusException "
 614                             + "attempting to promote then demote PDP={}, message=",
 615                             myPdp.getPdpId(), e1);
 616                 }
 617
 618             }
 619             waitTimerLastRunDate = currentTime.getDate();
 620             logger.debug("DesignatedWaiter.run (designatedPdp.getPdpId().equals(myPdp.getPdpId())) "
 621                             + "waitTimerLastRunDate = " + waitTimerLastRunDate);
 622             myPdp.setUpdatedDate(waitTimerLastRunDate);
 623             pdpsConnector.update(myPdp);
 624         }
 625     }
 626
 627     /**
 628      * Sanitize designated list.
 629      *
 630      * @param listOfDesignated list of designated pdps
 631      * @return list of drools pdps
 632      */
 633     public List<DroolsPdp> santizeDesignatedList(List<DroolsPdp> listOfDesignated) {
 634
 635         boolean containsDesignated = false;
 636         boolean containsHotStandby = false;
 637         List<DroolsPdp> listForRemoval = new ArrayList<>();
 638         for (DroolsPdp pdp : listOfDesignated) {
 639             logger.debug("DesignatedWaiter.run sanitizing: pdp = {}"
 640                             + " isDesignated = {}",pdp.getPdpId(), pdp.isDesignated());
 641             if (pdp.isDesignated()) {
 642                 containsDesignated = true;
 643             } else {
 644                 containsHotStandby = true;
 645                 listForRemoval.add(pdp);
 646             }
 647         }
 648         if (containsDesignated && containsHotStandby) {
 649             //remove the hot standby from the list
 650             listOfDesignated.removeAll(listForRemoval);
 651         }
 652         return listOfDesignated;
 653     }
 654
 655     /**
 656      * Compute most recent primary.
 657      *
 658      * @param pdps collection of pdps
 659      * @param listOfDesignated list of designated pdps
 660      * @return drools pdp object
 661      */
 662     public DroolsPdp computeMostRecentPrimary(Collection<DroolsPdp> pdps, List<DroolsPdp> listOfDesignated) {
 663         boolean containsDesignated = listOfDesignated.stream().anyMatch(DroolsPdp::isDesignated);
 664
 665         DroolsPdp mostRecentPrimary = new DroolsPdpImpl(null, true, 1, new Date(0));
 666         mostRecentPrimary.setSite(null);
 667         logger.debug("DesignatedWaiter.run listOfDesignated.size() = {}", listOfDesignated.size());
 668
 669         if (listOfDesignated.size() <= 1) {
 670             logger.debug("DesignatedWainter.run: listOfDesignated.size <=1");
 671             //Only one or none is designated or hot standby.  Choose the latest designated date
 672             mostRecentPrimary = getLatestDesignated(pdps, mostRecentPrimary);
 673
 674         } else if (listOfDesignated.size() == pdps.size()) {
 675             logger.debug("DesignatedWainter.run: listOfDesignated.size = pdps.size() which is {}", pdps.size());
 676             //They are all designated or all hot standby.
 677             mostRecentPrimary = getBestDesignated(pdps, containsDesignated);
 678
 679         } else {
 680             logger.debug("DesignatedWainter.run: Some but not all are designated or hot standby. ");
 681             logger.debug("DesignatedWainter.run: containsDesignated = {}", containsDesignated);
 682             //Some but not all are designated or hot standby.
 683             if (containsDesignated) {
 684                 /*
 685                  * The list only contains designated.  This is a problem.  It is most likely a race
 686                  * condition that resulted in two thinking they should be designated. Choose the
 687                  * site with the latest designated date for the pdp not included on the designated list.
 688                  * This should be the site that had the last designation before this race condition
 689                  * occurred.
 690                  */
 691                 mostRecentPrimary = getLatestUndesignated(pdps, mostRecentPrimary, listOfDesignated);
 692
 693             } else {
 694                 //The list only contains hot standby. Choose the site of the latest designated date
 695                 mostRecentPrimary = getLatestDesignated(pdps, mostRecentPrimary);
 696             }
 697         }
 698         return mostRecentPrimary;
 699     }
 700
 701     private DroolsPdp getBestDesignated(Collection<DroolsPdp> pdps, boolean containsDesignated) {
 702         DroolsPdp mostRecentPrimary;
 703         mostRecentPrimary = null;
 704         for (DroolsPdp pdp : pdps) {
 705             if (mostRecentPrimary == null) {
 706                 mostRecentPrimary = pdp;
 707                 continue;
 708             }
 709             if (containsDesignated) { //Choose the site of the first designated date
 710                 if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) < 0) {
 711                     mostRecentPrimary = pdp;
 712                     logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 713                 }
 714             } else { //Choose the site with the latest designated date
 715                 if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 716                     mostRecentPrimary = pdp;
 717                     logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 718                 }
 719             }
 720         }
 721         return mostRecentPrimary;
 722     }
 723
 724     private DroolsPdp getLatestUndesignated(Collection<DroolsPdp> pdps, DroolsPdp mostRecentPrimary,
 725                     List<DroolsPdp> listOfDesignated) {
 726         for (DroolsPdp pdp : pdps) {
 727             if (listOfDesignated.contains(pdp)) {
 728                 continue; //Don't consider this entry
 729             }
 730             if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 731                 mostRecentPrimary = pdp;
 732                 logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 733             }
 734         }
 735         return mostRecentPrimary;
 736     }
 737
 738     private DroolsPdp getLatestDesignated(Collection<DroolsPdp> pdps, DroolsPdp mostRecentPrimary) {
 739         for (DroolsPdp pdp : pdps) {
 740             logger.debug("DesignatedWaiter.run pdp = {}"
 741                             + " pdp.getDesignatedDate() = {}",
 742                             pdp.getPdpId(), pdp.getDesignatedDate());
 743             if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 744                 mostRecentPrimary = pdp;
 745                 logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 746             }
 747         }
 748         return mostRecentPrimary;
 749     }
 750
 751     /**
 752      * Compue designated pdp.
 753      *
 754      * @param listOfDesignated list of designated pdps
 755      * @param mostRecentPrimary most recent primary pdpd
 756      * @return drools pdp object
 757      */
 758     public DroolsPdp computeDesignatedPdp(List<DroolsPdp> listOfDesignated, DroolsPdp mostRecentPrimary) {
 759         if (listOfDesignated.isEmpty()) {
 760             logger.debug("\nDesignatedWaiter.run: myPdp: {} listOfDesignated is: EMPTY.", myPdp.getPdpId());
 761             return null;
 762         }
 763
 764         if (listOfDesignated.size() == 1) {
 765             logger.debug("\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 766                             + "has ONE entry. PDP ID: {}", myPdp.getPdpId(), listOfDesignated.get(0).getPdpId());
 767             return listOfDesignated.get(0);
 768         }
 769
 770         logger.debug("DesignatedWaiter.run: myPdp: {} listOfDesignated.size(): {}", myPdp.getPdpId(),
 771                         listOfDesignated.size());
 772         DesignatedData data = new DesignatedData();
 773         for (DroolsPdp pdp : listOfDesignated) {
 774             DroolsPdp rejectedPdp;
 775
 776             // We need to determine if another PDP is the lowest priority
 777             if (nullSafeEquals(pdp.getSite(), mostRecentPrimary.getSite())) {
 778                 rejectedPdp = data.compareSameSite(pdp);
 779             } else {
 780                 rejectedPdp = data.compareDifferentSite(pdp);
 781             }
 782             // If the rejectedPdp is myPdp, we need to stand it down and demote it.  Each pdp is responsible
 783             // for demoting itself
 784             if (rejectedPdp != null && nullSafeEquals(rejectedPdp.getPdpId(),myPdp.getPdpId())) {
 785                 logger.debug("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated myPdp ID: {}"
 786                                 + " is NOT the lowest priority.  Executing stateManagement.demote()\n\n",
 787                                 myPdp.getPdpId(),
 788                                 myPdp.getPdpId());
 789                 // We found that myPdp is on the listOfDesignated and it is not the lowest priority
 790                 // So, we must demote it
 791                 demoteMyPdp();
 792             }
 793         } //end: for(DroolsPdp pdp : listOfDesignated)
 794
 795         DroolsPdp lowestPriorityPdp = data.getLowestPriority();
 796
 797         //now we have a valid value for lowestPriorityPdp
 798         logger.debug("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 799                         + "found the LOWEST priority pdp ID: {} "
 800                         + " It is now the designatedPpd from the perspective of myPdp ID: {} \n\n",
 801                         myPdp.getPdpId(), lowestPriorityPdp.getPdpId(), myPdp);
 802         return lowestPriorityPdp;
 803
 804     }
 805
 806     private class DesignatedData {
 807         private DroolsPdp lowestPrioritySameSite = null;
 808         private DroolsPdp lowestPriorityDifferentSite = null;
 809
 810         private DroolsPdp compareSameSite(DroolsPdp pdp) {
 811             if (lowestPrioritySameSite == null) {
 812                 if (lowestPriorityDifferentSite != null) {
 813                     //we need to reject lowestPriorityDifferentSite
 814                     DroolsPdp rejectedPdp = lowestPriorityDifferentSite;
 815                     lowestPriorityDifferentSite = pdp;
 816                     return rejectedPdp;
 817                 }
 818                 lowestPrioritySameSite = pdp;
 819                 return null;
 820             } else {
 821                 if (pdp.getPdpId().equals((lowestPrioritySameSite.getPdpId()))) {
 822                     return null;//nothing to compare
 823                 }
 824                 if (pdp.comparePriority(lowestPrioritySameSite) < 0) {
 825                     logger.debug("\nDesignatedWaiter.run: myPdp {}  listOfDesignated pdp ID: {}"
 826                                     + " has lower priority than pdp ID: {}",myPdp.getPdpId(), pdp.getPdpId(),
 827                                     lowestPrioritySameSite.getPdpId());
 828                     //we need to reject lowestPrioritySameSite
 829                     DroolsPdp rejectedPdp = lowestPrioritySameSite;
 830                     lowestPrioritySameSite = pdp;
 831                     return rejectedPdp;
 832                 } else {
 833                     //we need to reject pdp and keep lowestPrioritySameSite
 834                     logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {} "
 835                                     + " has higher priority than pdp ID: {}", myPdp.getPdpId(),pdp.getPdpId(),
 836                                     lowestPrioritySameSite.getPdpId());
 837                     return pdp;
 838                 }
 839             }
 840         }
 841
 842         private DroolsPdp compareDifferentSite(DroolsPdp pdp) {
 843             if (lowestPrioritySameSite != null) {
 844                 //if we already have a candidate for same site, we don't want to bother with different sites
 845                 return pdp;
 846             } else {
 847                 if (lowestPriorityDifferentSite == null) {
 848                     lowestPriorityDifferentSite = pdp;
 849                     return null;
 850                 }
 851                 if (pdp.getPdpId().equals((lowestPriorityDifferentSite.getPdpId()))) {
 852                     return null;//nothing to compare
 853                 }
 854                 if (pdp.comparePriority(lowestPriorityDifferentSite) < 0) {
 855                     logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 856                                     + " has lower priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 857                                     lowestPriorityDifferentSite.getPdpId());
 858                     //we need to reject lowestPriorityDifferentSite
 859                     DroolsPdp rejectedPdp = lowestPriorityDifferentSite;
 860                     lowestPriorityDifferentSite = pdp;
 861                     return rejectedPdp;
 862                 } else {
 863                     //we need to reject pdp and keep lowestPriorityDifferentSite
 864                     logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 865                                     + " has higher priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 866                                     lowestPriorityDifferentSite.getPdpId());
 867                     return pdp;
 868                 }
 869             }
 870         }
 871
 872         private DroolsPdp getLowestPriority() {
 873             return (lowestPrioritySameSite != null ? lowestPrioritySameSite : lowestPriorityDifferentSite);
 874         }
 875     }
 876
 877     private void demoteMyPdp() {
 878         try {
 879             //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 880             myPdp.setDesignated(false);
 881             pdpsConnector.setDesignated(myPdp, false);
 882             isDesignated = false;
 883             String standbyStatus = stateManagementFeature.getStandbyStatus();
 884             if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 885                     || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 886                 /*
 887                  * Only call demote if it is not already in the right state.  Don't worry about
 888                  * synching the lower level topic endpoint states.  That is done by the
 889                  * refreshStateAudit.
 890                  */
 891                 stateManagementFeature.demote();
 892             }
 893         } catch (Exception e) {
 894             myPdp.setDesignated(false);
 895             pdpsConnector.setDesignated(myPdp, false);
 896             isDesignated = false;
 897             logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 898                     + "demote myPdp {} myPdp.getPdpId(), message= {}", myPdp.getPdpId(),
 899                     e);
 900         }
 901     }
 902
 903     private class TimerUpdateClass extends TimerTask {
 904
 905         @Override
 906         public void run() {
 907             try {
 908                 logger.debug("TimerUpdateClass.run: entry");
 909                 checkWaitTimer();
 910             } catch (Exception e) {
 911                 logger.error("TimerUpdateClass.run caught an unexpected exception: ", e);
 912             }
 913             logger.debug("TimerUpdateClass.run.exit");
 914         }
 915     }
 916
 917     @Override
 918     public void checkThreadStatus() {
 919         checkWaitTimer();
 920     }
 921
 922     private void checkWaitTimer() {
 923         synchronized (checkWaitTimerLock) {
 924             try {
 925                 logger.debug("checkWaitTimer: entry");
 926                 Date now = currentTime.getDate();
 927                 long nowMs = now.getTime();
 928                 long waitTimerMs = waitTimerLastRunDate.getTime();
 929
 930                 //give it 10 times leeway
 931                 if ((nowMs - waitTimerMs)  > 10 * pdpUpdateInterval) {
 932                     if (allSeemsWell == null || allSeemsWell) {
 933                         allSeemsWell = false;
 934                         logger.debug("checkWaitTimer: calling allSeemsWell with ALLNOTWELL param");
 935                         stateManagementFeature.allSeemsWell(this.getClass().getName(),
 936                                 StateManagementFeatureApiConstants.ALLNOTWELL_STATE,
 937                                 "DesignationWaiter/ElectionHandler has STALLED");
 938                     }
 939                     logger.error("checkWaitTimer: nowMs - waitTimerMs = {}"
 940                             + ", exceeds 10* pdpUpdateInterval = {}"
 941                             + " DesignationWaiter is STALLED!", (nowMs - waitTimerMs), (10 * pdpUpdateInterval));
 942                 } else if (allSeemsWell == null || !allSeemsWell) {
 943                     allSeemsWell = true;
 944                     stateManagementFeature.allSeemsWell(this.getClass().getName(),
 945                             StateManagementFeatureApiConstants.ALLSEEMSWELL_STATE,
 946                             "DesignationWaiter/ElectionHandler has RESUMED");
 947                     logger.info("DesignationWaiter/ElectionHandler has RESUMED");
 948                 }
 949                 logger.debug("checkWaitTimer: exit");
 950             } catch (Exception e) {
 951                 logger.error("checkWaitTimer: caught unexpected exception: ", e);
 952             }
 953         }
 954     }
 955
 956     private long getDWaiterStartMs() {
 957         Date now = currentTime.getDate();
 958
 959         // Retrieve the ms since the epoch
 960         long nowMs = now.getTime();
 961
 962         // Time since the end of the last pdpUpdateInterval multiple
 963         long nowModMs = nowMs % pdpUpdateInterval;
 964
 965         // Time to the start of the next pdpUpdateInterval multiple
 966         long startMs = 2 * pdpUpdateInterval - nowModMs;
 967
 968         // Give the start time a minimum of a 5 second cushion
 969         if (startMs < 5000) {
 970             // Start at the beginning  of following interval
 971             startMs = pdpUpdateInterval + startMs;
 972         }
 973         return startMs;
 974     }
 975
 976     private boolean nullSafeEquals(Object one, Object two) {
 977         if (one == null && two == null) {
 978             return true;
 979         }
 980         if (one != null && two != null) {
 981             return one.equals(two);
 982         }
 983         return false;
 984     }
 985
 986     public String getPdpdNowActive() {
 987         return pdpdNowActive;
 988     }
 989
 990     public String getPdpdLastActive() {
 991         return pdpdLastActive;
 992     }
 993 }