gerrit.onap Code Review - policy/drools-pdp.git/blob

   1 /*
   2  * ============LICENSE_START=======================================================
   3  * feature-active-standby-management
   4  * ================================================================================
   5  * Copyright (C) 2017-2021 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.policy.drools.activestandby;
  22
  23 import java.util.ArrayList;
  24 import java.util.Collection;
  25 import java.util.Date;
  26 import java.util.List;
  27 import java.util.Objects;
  28 import java.util.TimerTask;
  29 import lombok.Getter;
  30 import lombok.Setter;
  31 import org.onap.policy.common.im.MonitorTime;
  32 import org.onap.policy.common.im.StateManagement;
  33 import org.onap.policy.common.utils.time.CurrentTime;
  34 import org.onap.policy.drools.statemanagement.StateManagementFeatureApi;
  35 import org.onap.policy.drools.statemanagement.StateManagementFeatureApiConstants;
  36 import org.slf4j.Logger;
  37 import org.slf4j.LoggerFactory;
  38
  39 public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
  40     private static final String RUN_PRIMARY_MSG = "DesignatedWaiter.run mostRecentPrimary = {}";
  41
  42     // get an instance of logger
  43     private static final Logger  logger = LoggerFactory.getLogger(DroolsPdpsElectionHandler.class);
  44
  45     /*
  46      * Must be static, so it can be referenced by JpaDroolsPdpsConnector,
  47      * without requiring a reference to the election handler instantiation.
  48      */
  49     private static DroolsPdp myPdp;
  50
  51     @Setter
  52     private static boolean unitTesting = false;
  53     @Setter
  54     private static boolean stalled = false;
  55
  56     private DroolsPdpsConnector pdpsConnector;
  57     private Object checkWaitTimerLock = new Object();
  58     private Object designationWaiterLock = new Object();
  59
  60     private Date waitTimerLastRunDate;
  61
  62     // The interval between runs of the DesignationWaiter
  63     private int pdpUpdateInterval;
  64
  65     private volatile boolean isDesignated;
  66
  67     @Getter
  68     private String pdpdNowActive;
  69     @Getter
  70     private String pdpdLastActive;
  71
  72     /*
  73      * Start allSeemsWell with a value of null so that, on the first run
  74      * of the checkWaitTimer it will set the value in IntegrityMonitor
  75      * regardless of whether it needs to be set to true or false.
  76      */
  77     private Boolean allSeemsWell = null;
  78
  79     private StateManagementFeatureApi stateManagementFeature;
  80
  81     private final CurrentTime currentTime = MonitorTime.getInstance();
  82
  83     /**
  84      * Constructor.
  85      *
  86      * @param pdps connectors
  87      * @param myPdp pdp
  88      */
  89     public DroolsPdpsElectionHandler(DroolsPdpsConnector pdps, DroolsPdp myPdp) {
  90         if (pdps == null) {
  91             logger.error("DroolsPdpsElectinHandler(): pdpsConnector==null");
  92             throw new IllegalArgumentException("DroolsPdpsElectinHandler(): pdpsConnector==null");
  93         }
  94         if (myPdp == null) {
  95             logger.error("DroolsPdpsElectinHandler(): droolsPdp==null");
  96             throw new IllegalArgumentException("DroolsPdpsElectinHandler(): DroolsPdp==null");
  97         }
  98
  99         pdpdNowActive = null;
 100         pdpdLastActive = null;
 101         this.pdpsConnector = pdps;
 102         setMyPdp(myPdp);
 103         this.isDesignated = false;
 104
 105         // The interval between checks of the DesignationWaiter to be sure it is running.
 106         var pdpCheckInterval = 3000;
 107         try {
 108             pdpCheckInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(
 109                     ActiveStandbyProperties.PDP_CHECK_INVERVAL));
 110         } catch (Exception e) {
 111             logger.error("Could not get pdpCheckInterval property. Using default {}", pdpCheckInterval, e);
 112         }
 113         pdpUpdateInterval = 2000;
 114         try {
 115             pdpUpdateInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(
 116                     ActiveStandbyProperties.PDP_UPDATE_INTERVAL));
 117         } catch (Exception e) {
 118             logger.error("Could not get pdpUpdateInterval property. Using default {} ", pdpUpdateInterval, e);
 119         }
 120
 121         var now = currentTime.getDate();
 122
 123         // Retrieve the ms since the epoch
 124         final long nowMs = now.getTime();
 125
 126         // Create the timer which will update the updateDate in DroolsPdpEntity table.
 127         // This is the heartbeat
 128         var updateWorker = Factory.getInstance().makeTimer();
 129
 130         // Schedule the TimerUpdateClass to run at 100 ms and run at pdpCheckInterval ms thereafter
 131         // NOTE: The first run of the TimerUpdateClass results in myPdp being added to the
 132         // drools droolsPdpEntity table.
 133         updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
 134
 135         // Create the timer which will run the election algorithm
 136         var waitTimer = Factory.getInstance().makeTimer();
 137
 138         // Schedule it to start in startMs ms
 139         // (so it will run after the updateWorker and run at pdpUpdateInterval ms thereafter
 140         long startMs = getDWaiterStartMs();
 141         var designationWaiter = new DesignationWaiter();
 142         waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
 143         waitTimerLastRunDate = new Date(nowMs + startMs);
 144
 145         //Get the StateManagementFeature instance
 146
 147         for (StateManagementFeatureApi feature : StateManagementFeatureApiConstants.getImpl().getList()) {
 148             if (feature.getResourceName().equals(myPdp.getPdpId())) {
 149                 logger.debug("DroolsPdpsElectionHandler: Found StateManagementFeature"
 150                                 + " with resourceName: {}", myPdp.getPdpId());
 151                 stateManagementFeature = feature;
 152                 break;
 153             }
 154         }
 155         if (stateManagementFeature == null) {
 156             logger.error("DroolsPdpsElectionHandler failed to initialize.  "
 157                     + "Unable to get instance of StateManagementFeatureApi "
 158                     + "with resourceID: {}", myPdp.getPdpId());
 159         }
 160     }
 161
 162     private static void setMyPdp(DroolsPdp myPdp) {
 163         DroolsPdpsElectionHandler.myPdp = myPdp;
 164     }
 165
 166     /**
 167      * When the JpaDroolsPdpsConnector.standDown() method is invoked, it needs
 168      * access to myPdp, so it can keep its designation status in sync with the
 169      * DB.
 170      *
 171      * @param designated is designated value
 172      */
 173     public static void setMyPdpDesignated(boolean designated) {
 174         logger.debug("setMyPdpDesignated: designated= {}", designated);
 175         myPdp.setDesignated(designated);
 176     }
 177
 178     private class DesignationWaiter extends TimerTask {
 179         // get an instance of logger
 180         private final Logger  logger = LoggerFactory.getLogger(DesignationWaiter.class);
 181
 182         @Override
 183         public void run() {
 184             try {
 185                 logger.debug("DesignatedWaiter.run: Entering");
 186
 187                 //This is for testing the checkWaitTimer
 188                 if (unitTesting && stalled) {
 189                     logger.debug("DesignatedWaiter.run: isUnitTesting = {} isStalled = {}",
 190                                     unitTesting, stalled);
 191                     return;
 192                 }
 193
 194                 synchronized (designationWaiterLock) {
 195
 196                     logger.debug("DesignatedWaiter.run: Entering synchronized block");
 197
 198                     //It is possible that multiple PDPs are designated lead.  So, we will make a list of all designated
 199                     //PDPs and then decide which one really should be designated at the end.
 200                     List<DroolsPdp> listOfDesignated = new ArrayList<>();
 201
 202                     Collection<DroolsPdp> pdps = pdpsConnector.getDroolsPdps();
 203
 204                     logger.debug("DesignatedWaiter.run: pdps.size= {}", pdps.size());
 205
 206                     //This is only true if all designated PDPs have failed
 207                     allPdpsFailed(pdps, listOfDesignated);
 208
 209                     /*
 210                      * We have checked the four combinations of isDesignated and isCurrent.  Where appropriate,
 211                      * we added the PDPs to the potential list of designated pdps
 212                      *
 213                      * We need to give priority to pdps on the same site that is currently being used
 214                      * First, however, we must sanitize the list of designated to make sure their are
 215                      * only designated members or non-designated members.  There should not be both in
 216                      * the list. Because there are real time delays, it is possible that both types could
 217                      * be on the list.
 218                      */
 219
 220                     listOfDesignated = santizeDesignatedList(listOfDesignated);
 221
 222                     /*
 223                      * We need to figure out the last pdp that was the primary so we can get the last site
 224                      * name and the last session numbers.  We need to create a "dummy" droolspdp since
 225                      * it will be used in later comparisons and cannot be null.
 226                      */
 227
 228                     DroolsPdp mostRecentPrimary = computeMostRecentPrimary(pdps, listOfDesignated);
 229
 230                     if (mostRecentPrimary != null) {
 231                         pdpdLastActive = mostRecentPrimary.getPdpId();
 232                     }
 233
 234
 235                     /*
 236                      * It is possible to get here with more than one pdp designated and providing service. This normally
 237                      * occurs when there is a race condition with multiple nodes coming up at the same time. If that is
 238                      * the case we must determine which one is the one that should be designated and which one should
 239                      * be demoted.
 240                      *
 241                      * It is possible to have 0, 1, 2 or more but not all, or all designated.
 242                      *   If we have one designated and current, we chose it and are done
 243                      *   If we have 2 or more, but not all, we must determine which one is in the same site as
 244                      *   the previously designated pdp.
 245                      */
 246                     DroolsPdp designatedPdp = computeDesignatedPdp(listOfDesignated, mostRecentPrimary);
 247
 248                     if (designatedPdp == null) {
 249                         logger.warn("WARNING: DesignatedWaiter.run: No viable PDP found to be Designated. "
 250                             + "designatedPdp still null.");
 251                         designateNoPdp();
 252                         return;
 253                     }
 254
 255                     pdpdNowActive = designatedPdp.getPdpId();
 256
 257                     if (pdpdNowActive.equals(myPdp.getPdpId())) {
 258                         logger.debug("DesignatedWaiter.run: designatedPdp is PDP={}", myPdp.getPdpId());
 259                         designateMyPdp();
 260                         return;
 261                     }
 262
 263                     isDesignated = false;
 264
 265                 } // end synchronized
 266                 logger.debug("DesignatedWaiter.run: myPdp: {}; Returning, isDesignated= {}",
 267                                 isDesignated, myPdp.getPdpId());
 268
 269                 var tmpDate = currentTime.getDate();
 270                 logger.debug("DesignatedWaiter.run (end of run) waitTimerLastRunDate = {}", tmpDate);
 271
 272                 waitTimerLastRunDate = tmpDate;
 273                 myPdp.setUpdatedDate(waitTimerLastRunDate);
 274                 pdpsConnector.update(myPdp);
 275
 276             } catch (Exception e) {
 277                 logger.error("DesignatedWaiter.run caught an unexpected exception: ", e);
 278             }
 279         } // end run
 280
 281         private void allPdpsFailed(Collection<DroolsPdp> pdps, List<DroolsPdp> listOfDesignated) {
 282             boolean designatedPdpHasFailed = pdpsConnector.hasDesignatedPdpFailed(pdps);
 283             logger.debug("DesignatedWaiter.run: designatedPdpHasFailed= {}", designatedPdpHasFailed);
 284             for (DroolsPdp pdp : pdps) {
 285                 logger.debug("DesignatedWaiter.run: evaluating pdp ID: {}", pdp.getPdpId());
 286
 287                 /*
 288                  * Note: side effect of isPdpCurrent is that any stale but
 289                  * designated PDPs will be marked as un-designated.
 290                  */
 291                 boolean isCurrent = pdpsConnector.isPdpCurrent(pdp);
 292
 293                 /*
 294                  * We can't use stateManagement.getStandbyStatus() here, because
 295                  * we need the standbyStatus, not for this PDP, but for the PDP
 296                  * being processed by this loop iteration.
 297                  */
 298                 String standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 299                 if (standbyStatus == null) {
 300                     // Treat this case as a cold standby -- if we
 301                     // abort here, no sessions will be created in a
 302                     // single-node test environment.
 303                     standbyStatus = StateManagement.COLD_STANDBY;
 304                 }
 305                 logger.debug("DesignatedWaiter.run: PDP= {},  isCurrent= {}", pdp.getPdpId(), isCurrent);
 306
 307                 adjustPdp(pdp, isCurrent, designatedPdpHasFailed, standbyStatus, listOfDesignated);
 308
 309
 310             } // end pdps loop
 311         }
 312
 313         private void adjustPdp(DroolsPdp pdp, boolean isCurrent, boolean designatedPdpHasFailed, String standbyStatus,
 314                         List<DroolsPdp> listOfDesignated) {
 315             /*
 316              * There are 4 combinations of isDesignated and isCurrent.  We will examine each one in-turn
 317              * and evaluate the each pdp in the list of pdps against each combination.
 318              */
 319             if (pdp.isDesignated()) {
 320                 /*
 321                  * This is the first combination of isDesignated and isCurrent
 322                  */
 323                 if (isCurrent) {
 324                     pdpDesignatedCurrent(pdp, standbyStatus, listOfDesignated);
 325
 326                 /*
 327                  * The second combination of isDesignated and isCurrent
 328                  *
 329                  * PDP is designated but not current; it has failed.
 330                  * So we stand it down (it doesn't matter what
 331                  * its standbyStatus is). None of these go on the list.
 332                  */
 333                 } else {
 334                     logger.debug("INFO: DesignatedWaiter.run: PDP= {} is currently "
 335                                     + "designated but is not current; "
 336                                     + "it has failed.  Standing down.  standbyStatus= {}",
 337                                     pdp.getPdpId(), standbyStatus);
 338                     pdpDesignatedNotCurrent(pdp);
 339                 }
 340
 341             } else {
 342                 // NOT designated
 343
 344
 345                 /*
 346                  * The third combination of isDesignated and isCurrent
 347                  * /*
 348                  * If a PDP is not currently designated but is providing service
 349                  * (erroneous, but recoverable) or hot standby
 350                  * we can add it to the list of possible designated if all the designated have failed
 351                  */
 352                 if (isCurrent) {
 353                     pdpNotDesignatedCurrent(pdp, designatedPdpHasFailed, standbyStatus,
 354                                     listOfDesignated);
 355
 356                 /*
 357                  * The fourth combination of isDesignated and isCurrent
 358                  *
 359                  * We are not going to put any of these on the list since it appears they have failed.
 360                  *
 361                  */
 362                 } else {
 363                     logger.debug("INFO: DesignatedWaiter.run: PDP= {} "
 364                                     + "designated= {}, current= {}, "
 365                                     + "designatedPdpHasFailed= {}, "
 366                                     + "standbyStatus= {}", pdp.getPdpId(),
 367                                     pdp.isDesignated(), false, designatedPdpHasFailed, standbyStatus);
 368                     pdpNotDesignatedNotCurrent(pdp, standbyStatus);
 369                 }
 370             }
 371         }
 372
 373         private void pdpDesignatedCurrent(DroolsPdp pdp, String standbyStatus, List<DroolsPdp> listOfDesignated) {
 374             //It is current, but it could have a standbystatus=coldstandby / hotstandby
 375             //If so, we need to stand it down and demote it
 376             if (!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)) {
 377                 if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 378                     logger.debug("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 379                                     + "butstandbystatus is not providingservice. "
 380                                     + " Executing stateManagement.demote()" + "\n\n", myPdp.getPdpId());
 381                     // So, we must demote it
 382                     try {
 383                         demoteMyPdp(pdp, standbyStatus);
 384                     } catch (Exception e) {
 385                         logger.error("DesignatedWaiter.run: myPdp: {} "
 386                                 + "Caught Exception attempting to demote myPdp,"
 387                                 + "message= {}", myPdp.getPdpId(), e);
 388                     }
 389                 } else {
 390                     // Don't demote a remote PDP that is current.  It should catch itself
 391                     logger.debug("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 392                                     + "but standbystatus is not providingservice. "
 393                                     + " Cannot execute stateManagement.demote() "
 394                                     + "since it it is not myPdp\n\n",
 395                                     myPdp.getPdpId());
 396                 }
 397
 398             } else {
 399                 // If we get here, it is ok to be on the list
 400                 logger.debug("DesignatedWaiter.run: PDP= {} is designated, "
 401                                 + "current and {} Noting PDP as "
 402                                 + "designated, standbyStatus= {}",
 403                                 pdp.getPdpId(), standbyStatus, standbyStatus);
 404                 listOfDesignated.add(pdp);
 405             }
 406         }
 407
 408         private void demoteMyPdp(DroolsPdp pdp, String standbyStatus) throws Exception {
 409             /*
 410              * Keep the order like this. StateManagement is last since it triggers
 411              * controller shutdown. This will change isDesignated and it can enter another
 412              * if-combination below
 413              */
 414             pdpsConnector.standDownPdp(pdp.getPdpId());
 415             myPdp.setDesignated(false);
 416             isDesignated = false;
 417             if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 418                     || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 419                 /*
 420                  * Only demote it if it appears it has not already been demoted. Don't worry
 421                  * about synching with the topic endpoint states.  That is done by the
 422                  * refreshStateAudit
 423                  */
 424                 stateManagementFeature.demote();
 425             }
 426         }
 427
 428         private void pdpDesignatedNotCurrent(DroolsPdp pdp) {
 429             /*
 430              * Changes designated to 0 but it is still potentially providing service.
 431              * Will affect isDesignated, so, it can enter an if-combination below
 432              */
 433             pdpsConnector.standDownPdp(pdp.getPdpId());
 434
 435             //need to change standbystatus to coldstandby
 436             if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 437                 logger.debug("\n\nDesignatedWaiter.run: myPdp {} is not Current. "
 438                                 + " Executing stateManagement.disableFailed()\n\n", myPdp.getPdpId());
 439                 // We found that myPdp is designated but not current
 440                 // So, we must cause it to disableFail
 441                 try {
 442                     myPdp.setDesignated(false);
 443                     pdpsConnector.setDesignated(myPdp, false);
 444                     isDesignated = false;
 445                     stateManagementFeature.disableFailed();
 446                 } catch (Exception e) {
 447                     logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception "
 448                             + "attempting to disableFail myPdp {}, message= {}",
 449                             myPdp.getPdpId(), myPdp.getPdpId(), e);
 450                 }
 451             } else { //it is a remote PDP that is failed
 452                 logger.debug("\n\nDesignatedWaiter.run: PDP {} is not Current. "
 453                                 + " Executing stateManagement.disableFailed(otherResourceName)\n\n",
 454                                 pdp.getPdpId());
 455                 // We found a PDP is designated but not current
 456                 // We already called standdown(pdp) which will change designated to false
 457                 // Now we need to disableFail it to get its states in synch.  The standbyStatus
 458                 // should equal coldstandby
 459                 try {
 460                     stateManagementFeature.disableFailed(pdp.getPdpId());
 461                 } catch (Exception e) {
 462                     logger.error("DesignatedWaiter.run: for PDP {}  Caught Exception attempting to "
 463                             + "disableFail({}), message= {}",
 464                             pdp.getPdpId(), pdp.getPdpId(), e);
 465                 }
 466
 467             }
 468         }
 469
 470         private void pdpNotDesignatedCurrent(DroolsPdp pdp, boolean designatedPdpHasFailed, String standbyStatus,
 471                         List<DroolsPdp> listOfDesignated) {
 472             if (!(StateManagement.HOT_STANDBY.equals(standbyStatus)
 473                     || StateManagement.COLD_STANDBY.equals(standbyStatus))) {
 474                 logger.debug("\n\nDesignatedWaiter.run: PDP {}"
 475                                 + " is NOT designated but IS current and"
 476                                 + " has a standbystatus= {}", pdp.getPdpId(), standbyStatus);
 477                 // Since it is current, we assume it can adjust its own state.
 478                 // We will demote if it is myPdp
 479                 if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 480                     //demote it
 481                     logger.debug("DesignatedWaiter.run: PDP {} going to "
 482                                     + "setDesignated = false and calling stateManagement.demote",
 483                                     pdp.getPdpId());
 484                     try {
 485                         //Keep the order like this.
 486                         //StateManagement is last since it triggers controller shutdown
 487                         pdpsConnector.setDesignated(myPdp, false);
 488                         myPdp.setDesignated(false);
 489                         isDesignated = false;
 490                         //This is definitely not a redundant call.
 491                         //It is attempting to correct a problem
 492                         stateManagementFeature.demote();
 493                         //recheck the standbystatus
 494                         standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 495                     } catch (Exception e) {
 496                         logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception "
 497                                 + "attempting to demote myPdp {}, message = {}",  myPdp.getPdpId(),
 498                                 myPdp.getPdpId(), e);
 499                     }
 500
 501                 }
 502             }
 503             if (StateManagement.HOT_STANDBY.equals(standbyStatus) && designatedPdpHasFailed) {
 504                 //add it to the list
 505                 logger.debug("INFO: DesignatedWaiter.run: PDP= {}"
 506                                 + " is not designated but is {} and designated PDP "
 507                                 + "has failed.  standbyStatus= {}", pdp.getPdpId(),
 508                                 standbyStatus, standbyStatus);
 509                 listOfDesignated.add(pdp);
 510             }
 511         }
 512
 513         private void pdpNotDesignatedNotCurrent(DroolsPdp pdp, String standbyStatus) {
 514             if (StateManagement.COLD_STANDBY.equals(standbyStatus)) {
 515                 return;
 516             }
 517
 518             //stand it down
 519             //disableFail it
 520             pdpsConnector.standDownPdp(pdp.getPdpId());
 521             if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 522                 /*
 523                  * I don't actually know how this condition could
 524                  * happen, but if it did, we would want to declare it
 525                  * failed.
 526                  */
 527                 logger.debug("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 528                                 + " Executing stateManagement.disableFailed()\n\n",
 529                                 myPdp.getPdpId());
 530                 // So, we must disableFail it
 531                 try {
 532                     //Keep the order like this.
 533                     //StateManagement is last since it triggers controller shutdown
 534                     pdpsConnector.setDesignated(myPdp, false);
 535                     myPdp.setDesignated(false);
 536                     isDesignated = false;
 537                     stateManagementFeature.disableFailed();
 538                 } catch (Exception e) {
 539                     logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 540                             + "disableFail myPdp {}, message= {}",
 541                             myPdp.getPdpId(), myPdp.getPdpId(), e);
 542                 }
 543             } else { //it is remote
 544                 logger.debug("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 545                                 + " Executing stateManagement.disableFailed({})\n\n",
 546                                 myPdp.getPdpId(), pdp.getPdpId());
 547                 // We already called standdown(pdp) which will change designated to false
 548                 // Now we need to disableFail it to get its states in sync.
 549                 // StandbyStatus = coldstandby
 550                 try {
 551                     stateManagementFeature.disableFailed(pdp.getPdpId());
 552                 } catch (Exception e) {
 553                     logger.error("DesignatedWaiter.run: for PDP {}"
 554                             + " Caught Exception attempting to disableFail({})"
 555                             + ", message=", pdp.getPdpId(), pdp.getPdpId(), e);
 556                 }
 557             }
 558         }
 559
 560         private void designateNoPdp() {
 561             // Just to be sure the parameters are correctly set
 562             myPdp.setDesignated(false);
 563             pdpsConnector.setDesignated(myPdp, false);
 564             isDesignated = false;
 565
 566             waitTimerLastRunDate = currentTime.getDate();
 567             logger.debug("DesignatedWaiter.run (designatedPdp == null) waitTimerLastRunDate = {}",
 568                             waitTimerLastRunDate);
 569             myPdp.setUpdatedDate(waitTimerLastRunDate);
 570             pdpsConnector.update(myPdp);
 571         }
 572
 573         private void designateMyPdp() {
 574             /*
 575              * update function expects myPdp.isDesignated to be true.
 576              */
 577             try {
 578                 //Keep the order like this.  StateManagement is last since it triggers controller init
 579                 myPdp.setDesignated(true);
 580                 myPdp.setDesignatedDate(currentTime.getDate());
 581                 pdpsConnector.setDesignated(myPdp, true);
 582                 isDesignated = true;
 583                 String standbyStatus = stateManagementFeature.getStandbyStatus();
 584                 if (!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)) {
 585                     /*
 586                      * Only call promote if it is not already in the right state.  Don't worry about
 587                      * synching the lower level topic endpoint states.  That is done by the
 588                      * refreshStateAudit.
 589                      */
 590                     stateManagementFeature.promote();
 591                 }
 592             } catch (Exception e) {
 593                 logger.error("ERROR: DesignatedWaiter.run: Caught Exception attempting to promote PDP={}"
 594                         + ", message=", myPdp.getPdpId(), e);
 595                 myPdp.setDesignated(false);
 596                 pdpsConnector.setDesignated(myPdp, false);
 597                 isDesignated = false;
 598                 //If you can't promote it, demote it
 599                 try {
 600                     String standbyStatus = stateManagementFeature.getStandbyStatus();
 601                     if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 602                             || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 603                         /*
 604                          * Only call demote if it is not already in the right state.  Don't worry about
 605                          * synching the lower level topic endpoint states.  That is done by the
 606                          * refreshStateAudit.
 607                          */
 608                         stateManagementFeature.demote();
 609                     }
 610                 } catch (Exception e1) {
 611                     logger.error("ERROR: DesignatedWaiter.run: Caught StandbyStatusException "
 612                             + "attempting to promote then demote PDP={}, message=",
 613                             myPdp.getPdpId(), e1);
 614                 }
 615
 616             }
 617             waitTimerLastRunDate = currentTime.getDate();
 618             logger.debug("DesignatedWaiter.run (designatedPdp.getPdpId().equals(myPdp.getPdpId())) "
 619                             + "waitTimerLastRunDate = {}", waitTimerLastRunDate);
 620             myPdp.setUpdatedDate(waitTimerLastRunDate);
 621             pdpsConnector.update(myPdp);
 622         }
 623     }
 624
 625     /**
 626      * Sanitize designated list.
 627      *
 628      * @param listOfDesignated list of designated pdps
 629      * @return list of drools pdps
 630      */
 631     public List<DroolsPdp> santizeDesignatedList(List<DroolsPdp> listOfDesignated) {
 632
 633         var containsDesignated = false;
 634         var containsHotStandby = false;
 635         List<DroolsPdp> listForRemoval = new ArrayList<>();
 636         for (DroolsPdp pdp : listOfDesignated) {
 637             logger.debug("DesignatedWaiter.run sanitizing: pdp = {}"
 638                             + " isDesignated = {}", pdp.getPdpId(), pdp.isDesignated());
 639             if (pdp.isDesignated()) {
 640                 containsDesignated = true;
 641             } else {
 642                 containsHotStandby = true;
 643                 listForRemoval.add(pdp);
 644             }
 645         }
 646         if (containsDesignated && containsHotStandby) {
 647             //remove the hot standby from the list
 648             listOfDesignated.removeAll(listForRemoval);
 649         }
 650         return listOfDesignated;
 651     }
 652
 653     /**
 654      * Compute most recent primary.
 655      *
 656      * @param pdps collection of pdps
 657      * @param listOfDesignated list of designated pdps
 658      * @return drools pdp object
 659      */
 660     public DroolsPdp computeMostRecentPrimary(Collection<DroolsPdp> pdps, List<DroolsPdp> listOfDesignated) {
 661         boolean containsDesignated = listOfDesignated.stream().anyMatch(DroolsPdp::isDesignated);
 662
 663         DroolsPdp mostRecentPrimary = new DroolsPdpImpl(null, true, 1, new Date(0));
 664         mostRecentPrimary.setSite(null);
 665         logger.debug("DesignatedWaiter.run listOfDesignated.size() = {}", listOfDesignated.size());
 666
 667         if (listOfDesignated.size() <= 1) {
 668             logger.debug("DesignatedWainter.run: listOfDesignated.size <=1");
 669             //Only one or none is designated or hot standby.  Choose the latest designated date
 670             mostRecentPrimary = getLatestDesignated(pdps, mostRecentPrimary);
 671
 672         } else if (listOfDesignated.size() == pdps.size()) {
 673             logger.debug("DesignatedWainter.run: listOfDesignated.size = pdps.size() which is {}", pdps.size());
 674             //They are all designated or all hot standby.
 675             mostRecentPrimary = getBestDesignated(pdps, containsDesignated);
 676
 677         } else {
 678             logger.debug("DesignatedWainter.run: Some but not all are designated or hot standby. ");
 679             logger.debug("DesignatedWainter.run: containsDesignated = {}", containsDesignated);
 680             //Some but not all are designated or hot standby.
 681             if (containsDesignated) {
 682                 /*
 683                  * The list only contains designated.  This is a problem.  It is most likely a race
 684                  * condition that resulted in two thinking they should be designated. Choose the
 685                  * site with the latest designated date for the pdp not included on the designated list.
 686                  * This should be the site that had the last designation before this race condition
 687                  * occurred.
 688                  */
 689                 mostRecentPrimary = getLatestUndesignated(pdps, mostRecentPrimary, listOfDesignated);
 690
 691             } else {
 692                 //The list only contains hot standby. Choose the site of the latest designated date
 693                 mostRecentPrimary = getLatestDesignated(pdps, mostRecentPrimary);
 694             }
 695         }
 696         return mostRecentPrimary;
 697     }
 698
 699     private DroolsPdp getBestDesignated(Collection<DroolsPdp> pdps, boolean containsDesignated) {
 700         DroolsPdp mostRecentPrimary;
 701         mostRecentPrimary = null;
 702         for (DroolsPdp pdp : pdps) {
 703             if (mostRecentPrimary == null) {
 704                 mostRecentPrimary = pdp;
 705                 continue;
 706             }
 707             if (containsDesignated) { //Choose the site of the first designated date
 708                 if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) < 0) {
 709                     mostRecentPrimary = pdp;
 710                     logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 711                 }
 712             } else { //Choose the site with the latest designated date
 713                 if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 714                     mostRecentPrimary = pdp;
 715                     logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 716                 }
 717             }
 718         }
 719         return mostRecentPrimary;
 720     }
 721
 722     private DroolsPdp getLatestUndesignated(Collection<DroolsPdp> pdps, DroolsPdp mostRecentPrimary,
 723                     List<DroolsPdp> listOfDesignated) {
 724         for (DroolsPdp pdp : pdps) {
 725             if (listOfDesignated.contains(pdp)) {
 726                 continue; //Don't consider this entry
 727             }
 728             if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 729                 mostRecentPrimary = pdp;
 730                 logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 731             }
 732         }
 733         return mostRecentPrimary;
 734     }
 735
 736     private DroolsPdp getLatestDesignated(Collection<DroolsPdp> pdps, DroolsPdp mostRecentPrimary) {
 737         for (DroolsPdp pdp : pdps) {
 738             logger.debug("DesignatedWaiter.run pdp = {}"
 739                             + " pdp.getDesignatedDate() = {}",
 740                             pdp.getPdpId(), pdp.getDesignatedDate());
 741             if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 742                 mostRecentPrimary = pdp;
 743                 logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 744             }
 745         }
 746         return mostRecentPrimary;
 747     }
 748
 749     /**
 750      * Compue designated pdp.
 751      *
 752      * @param listOfDesignated list of designated pdps
 753      * @param mostRecentPrimary most recent primary pdpd
 754      * @return drools pdp object
 755      */
 756     public DroolsPdp computeDesignatedPdp(List<DroolsPdp> listOfDesignated, DroolsPdp mostRecentPrimary) {
 757         if (listOfDesignated.isEmpty()) {
 758             logger.debug("\nDesignatedWaiter.run: myPdp: {} listOfDesignated is: EMPTY.", myPdp.getPdpId());
 759             return null;
 760         }
 761
 762         if (listOfDesignated.size() == 1) {
 763             logger.debug("\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 764                             + "has ONE entry. PDP ID: {}", myPdp.getPdpId(), listOfDesignated.get(0).getPdpId());
 765             return listOfDesignated.get(0);
 766         }
 767
 768         logger.debug("DesignatedWaiter.run: myPdp: {} listOfDesignated.size(): {}", myPdp.getPdpId(),
 769                         listOfDesignated.size());
 770         var data = new DesignatedData();
 771         for (DroolsPdp pdp : listOfDesignated) {
 772             DroolsPdp rejectedPdp;
 773
 774             // We need to determine if another PDP is the lowest priority
 775             if (Objects.equals(pdp.getSite(), mostRecentPrimary.getSite())) {
 776                 rejectedPdp = data.compareSameSite(pdp);
 777             } else {
 778                 rejectedPdp = data.compareDifferentSite(pdp);
 779             }
 780             // If the rejectedPdp is myPdp, we need to stand it down and demote it.  Each pdp is responsible
 781             // for demoting itself
 782             if (rejectedPdp != null && Objects.equals(rejectedPdp.getPdpId(), myPdp.getPdpId())) {
 783                 logger.debug("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated myPdp ID: {}"
 784                                 + " is NOT the lowest priority.  Executing stateManagement.demote()\n\n",
 785                                 myPdp.getPdpId(),
 786                                 myPdp.getPdpId());
 787                 // We found that myPdp is on the listOfDesignated and it is not the lowest priority
 788                 // So, we must demote it
 789                 demoteMyPdp();
 790             }
 791         }
 792
 793         DroolsPdp lowestPriorityPdp = data.getLowestPriority();
 794
 795         //now we have a valid value for lowestPriorityPdp
 796         logger.debug("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 797                         + "found the LOWEST priority pdp ID: {} "
 798                         + " It is now the designatedPpd from the perspective of myPdp ID: {} \n\n",
 799                         myPdp.getPdpId(), lowestPriorityPdp.getPdpId(), myPdp);
 800         return lowestPriorityPdp;
 801
 802     }
 803
 804     private class DesignatedData {
 805         private DroolsPdp lowestPrioritySameSite = null;
 806         private DroolsPdp lowestPriorityDifferentSite = null;
 807
 808         private DroolsPdp compareSameSite(DroolsPdp pdp) {
 809             if (lowestPrioritySameSite == null) {
 810                 if (lowestPriorityDifferentSite != null) {
 811                     //we need to reject lowestPriorityDifferentSite
 812                     DroolsPdp rejectedPdp = lowestPriorityDifferentSite;
 813                     lowestPriorityDifferentSite = pdp;
 814                     return rejectedPdp;
 815                 }
 816                 lowestPrioritySameSite = pdp;
 817                 return null;
 818             } else {
 819                 if (pdp.getPdpId().equals((lowestPrioritySameSite.getPdpId()))) {
 820                     return null;    //nothing to compare
 821                 }
 822                 if (pdp.comparePriority(lowestPrioritySameSite) < 0) {
 823                     logger.debug("\nDesignatedWaiter.run: myPdp {}  listOfDesignated pdp ID: {}"
 824                                     + " has lower priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 825                                     lowestPrioritySameSite.getPdpId());
 826                     //we need to reject lowestPrioritySameSite
 827                     DroolsPdp rejectedPdp = lowestPrioritySameSite;
 828                     lowestPrioritySameSite = pdp;
 829                     return rejectedPdp;
 830                 } else {
 831                     //we need to reject pdp and keep lowestPrioritySameSite
 832                     logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {} "
 833                                     + " has higher priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 834                                     lowestPrioritySameSite.getPdpId());
 835                     return pdp;
 836                 }
 837             }
 838         }
 839
 840         private DroolsPdp compareDifferentSite(DroolsPdp pdp) {
 841             if (lowestPrioritySameSite != null) {
 842                 //if we already have a candidate for same site, we don't want to bother with different sites
 843                 return pdp;
 844             } else {
 845                 if (lowestPriorityDifferentSite == null) {
 846                     lowestPriorityDifferentSite = pdp;
 847                     return null;
 848                 }
 849                 if (pdp.getPdpId().equals((lowestPriorityDifferentSite.getPdpId()))) {
 850                     return null;    //nothing to compare
 851                 }
 852                 if (pdp.comparePriority(lowestPriorityDifferentSite) < 0) {
 853                     logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 854                                     + " has lower priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 855                                     lowestPriorityDifferentSite.getPdpId());
 856                     //we need to reject lowestPriorityDifferentSite
 857                     DroolsPdp rejectedPdp = lowestPriorityDifferentSite;
 858                     lowestPriorityDifferentSite = pdp;
 859                     return rejectedPdp;
 860                 } else {
 861                     //we need to reject pdp and keep lowestPriorityDifferentSite
 862                     logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 863                                     + " has higher priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 864                                     lowestPriorityDifferentSite.getPdpId());
 865                     return pdp;
 866                 }
 867             }
 868         }
 869
 870         private DroolsPdp getLowestPriority() {
 871             return (lowestPrioritySameSite != null ? lowestPrioritySameSite : lowestPriorityDifferentSite);
 872         }
 873     }
 874
 875     private void demoteMyPdp() {
 876         try {
 877             //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 878             myPdp.setDesignated(false);
 879             pdpsConnector.setDesignated(myPdp, false);
 880             isDesignated = false;
 881             String standbyStatus = stateManagementFeature.getStandbyStatus();
 882             if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 883                     || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 884                 /*
 885                  * Only call demote if it is not already in the right state.  Don't worry about
 886                  * synching the lower level topic endpoint states.  That is done by the
 887                  * refreshStateAudit.
 888                  */
 889                 stateManagementFeature.demote();
 890             }
 891         } catch (Exception e) {
 892             myPdp.setDesignated(false);
 893             pdpsConnector.setDesignated(myPdp, false);
 894             isDesignated = false;
 895             logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 896                     + "demote myPdp {} myPdp.getPdpId(), message= {}", myPdp.getPdpId(),
 897                     e);
 898         }
 899     }
 900
 901     private class TimerUpdateClass extends TimerTask {
 902
 903         @Override
 904         public void run() {
 905             try {
 906                 logger.debug("TimerUpdateClass.run: entry");
 907                 checkWaitTimer();
 908             } catch (Exception e) {
 909                 logger.error("TimerUpdateClass.run caught an unexpected exception: ", e);
 910             }
 911             logger.debug("TimerUpdateClass.run.exit");
 912         }
 913     }
 914
 915     @Override
 916     public void checkThreadStatus() {
 917         checkWaitTimer();
 918     }
 919
 920     private void checkWaitTimer() {
 921         synchronized (checkWaitTimerLock) {
 922             try {
 923                 logger.debug("checkWaitTimer: entry");
 924                 var now = currentTime.getDate();
 925                 long nowMs = now.getTime();
 926                 long waitTimerMs = waitTimerLastRunDate.getTime();
 927
 928                 //give it 10 times leeway
 929                 if ((nowMs - waitTimerMs)  > 10 * pdpUpdateInterval) {
 930                     if (allSeemsWell == null || allSeemsWell) {
 931                         allSeemsWell = false;
 932                         logger.debug("checkWaitTimer: calling allSeemsWell with ALLNOTWELL param");
 933                         stateManagementFeature.allSeemsWell(this.getClass().getName(),
 934                                 StateManagementFeatureApiConstants.ALLNOTWELL_STATE,
 935                                 "DesignationWaiter/ElectionHandler has STALLED");
 936                     }
 937                     logger.error("checkWaitTimer: nowMs - waitTimerMs = {}"
 938                             + ", exceeds 10* pdpUpdateInterval = {}"
 939                             + " DesignationWaiter is STALLED!", (nowMs - waitTimerMs), (10 * pdpUpdateInterval));
 940                 } else if (allSeemsWell == null || !allSeemsWell) {
 941                     allSeemsWell = true;
 942                     stateManagementFeature.allSeemsWell(this.getClass().getName(),
 943                             StateManagementFeatureApiConstants.ALLSEEMSWELL_STATE,
 944                             "DesignationWaiter/ElectionHandler has RESUMED");
 945                     logger.info("DesignationWaiter/ElectionHandler has RESUMED");
 946                 }
 947                 logger.debug("checkWaitTimer: exit");
 948             } catch (Exception e) {
 949                 logger.error("checkWaitTimer: caught unexpected exception: ", e);
 950             }
 951         }
 952     }
 953
 954     private long getDWaiterStartMs() {
 955         var now = currentTime.getDate();
 956
 957         // Retrieve the ms since the epoch
 958         long nowMs = now.getTime();
 959
 960         // Time since the end of the last pdpUpdateInterval multiple
 961         long nowModMs = nowMs % pdpUpdateInterval;
 962
 963         // Time to the start of the next pdpUpdateInterval multiple
 964         long startMs = 2 * pdpUpdateInterval - nowModMs;
 965
 966         // Give the start time a minimum of a 5 second cushion
 967         if (startMs < 5000) {
 968             // Start at the beginning  of following interval
 969             startMs = pdpUpdateInterval + startMs;
 970         }
 971         return startMs;
 972     }
 973 }