gerrit.onap Code Review - policy/drools-pdp.git/blob

   1 /*
   2  * ============LICENSE_START=======================================================
   3  * feature-active-standby-management
   4  * ================================================================================
   5  * Copyright (C) 2017-2021 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.policy.drools.activestandby;
  22
  23 import java.util.ArrayList;
  24 import java.util.Collection;
  25 import java.util.Date;
  26 import java.util.List;
  27 import java.util.Objects;
  28 import java.util.Timer;
  29 import java.util.TimerTask;
  30 import lombok.Getter;
  31 import lombok.Setter;
  32 import org.onap.policy.common.im.MonitorTime;
  33 import org.onap.policy.common.im.StateManagement;
  34 import org.onap.policy.common.utils.time.CurrentTime;
  35 import org.onap.policy.drools.statemanagement.StateManagementFeatureApi;
  36 import org.onap.policy.drools.statemanagement.StateManagementFeatureApiConstants;
  37 import org.slf4j.Logger;
  38 import org.slf4j.LoggerFactory;
  39
  40 public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
  41     private static final String RUN_PRIMARY_MSG = "DesignatedWaiter.run mostRecentPrimary = {}";
  42
  43     // get an instance of logger
  44     private static final Logger  logger = LoggerFactory.getLogger(DroolsPdpsElectionHandler.class);
  45
  46     /*
  47      * Must be static, so it can be referenced by JpaDroolsPdpsConnector,
  48      * without requiring a reference to the election handler instantiation.
  49      */
  50     private static DroolsPdp myPdp;
  51
  52     @Setter
  53     private static boolean unitTesting = false;
  54     @Setter
  55     private static boolean stalled = false;
  56
  57     private DroolsPdpsConnector pdpsConnector;
  58     private Object checkWaitTimerLock = new Object();
  59     private Object designationWaiterLock = new Object();
  60
  61     private Date waitTimerLastRunDate;
  62
  63     // The interval between runs of the DesignationWaiter
  64     private int pdpUpdateInterval;
  65
  66     private volatile boolean isDesignated;
  67
  68     @Getter
  69     private String pdpdNowActive;
  70     @Getter
  71     private String pdpdLastActive;
  72
  73     /*
  74      * Start allSeemsWell with a value of null so that, on the first run
  75      * of the checkWaitTimer it will set the value in IntegrityMonitor
  76      * regardless of whether it needs to be set to true or false.
  77      */
  78     private Boolean allSeemsWell = null;
  79
  80     private StateManagementFeatureApi stateManagementFeature;
  81
  82     private final CurrentTime currentTime = MonitorTime.getInstance();
  83
  84     /**
  85      * Constructor.
  86      *
  87      * @param pdps connectors
  88      * @param myPdp pdp
  89      */
  90     public DroolsPdpsElectionHandler(DroolsPdpsConnector pdps, DroolsPdp myPdp) {
  91         if (pdps == null) {
  92             logger.error("DroolsPdpsElectinHandler(): pdpsConnector==null");
  93             throw new IllegalArgumentException("DroolsPdpsElectinHandler(): pdpsConnector==null");
  94         }
  95         if (myPdp == null) {
  96             logger.error("DroolsPdpsElectinHandler(): droolsPdp==null");
  97             throw new IllegalArgumentException("DroolsPdpsElectinHandler(): DroolsPdp==null");
  98         }
  99
 100         pdpdNowActive = null;
 101         pdpdLastActive = null;
 102         this.pdpsConnector = pdps;
 103         setMyPdp(myPdp);
 104         this.isDesignated = false;
 105
 106         // The interval between checks of the DesignationWaiter to be sure it is running.
 107         int pdpCheckInterval = 3000;
 108         try {
 109             pdpCheckInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(
 110                     ActiveStandbyProperties.PDP_CHECK_INVERVAL));
 111         } catch (Exception e) {
 112             logger.error("Could not get pdpCheckInterval property. Using default {}", pdpCheckInterval, e);
 113         }
 114         pdpUpdateInterval = 2000;
 115         try {
 116             pdpUpdateInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(
 117                     ActiveStandbyProperties.PDP_UPDATE_INTERVAL));
 118         } catch (Exception e) {
 119             logger.error("Could not get pdpUpdateInterval property. Using default {} ", pdpUpdateInterval, e);
 120         }
 121
 122         Date now = currentTime.getDate();
 123
 124         // Retrieve the ms since the epoch
 125         final long nowMs = now.getTime();
 126
 127         // Create the timer which will update the updateDate in DroolsPdpEntity table.
 128         // This is the heartbeat
 129         Timer updateWorker = Factory.getInstance().makeTimer();
 130
 131         // Schedule the TimerUpdateClass to run at 100 ms and run at pdpCheckInterval ms thereafter
 132         // NOTE: The first run of the TimerUpdateClass results in myPdp being added to the
 133         // drools droolsPdpEntity table.
 134         updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
 135
 136         // Create the timer which will run the election algorithm
 137         Timer waitTimer = Factory.getInstance().makeTimer();
 138
 139         // Schedule it to start in startMs ms
 140         // (so it will run after the updateWorker and run at pdpUpdateInterval ms thereafter
 141         long startMs = getDWaiterStartMs();
 142         DesignationWaiter designationWaiter = new DesignationWaiter();
 143         waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
 144         waitTimerLastRunDate = new Date(nowMs + startMs);
 145
 146         //Get the StateManagementFeature instance
 147
 148         for (StateManagementFeatureApi feature : StateManagementFeatureApiConstants.getImpl().getList()) {
 149             if (feature.getResourceName().equals(myPdp.getPdpId())) {
 150                 logger.debug("DroolsPdpsElectionHandler: Found StateManagementFeature"
 151                                 + " with resourceName: {}", myPdp.getPdpId());
 152                 stateManagementFeature = feature;
 153                 break;
 154             }
 155         }
 156         if (stateManagementFeature == null) {
 157             logger.error("DroolsPdpsElectionHandler failed to initialize.  "
 158                     + "Unable to get instance of StateManagementFeatureApi "
 159                     + "with resourceID: {}", myPdp.getPdpId());
 160         }
 161     }
 162
 163     private static void setMyPdp(DroolsPdp myPdp) {
 164         DroolsPdpsElectionHandler.myPdp = myPdp;
 165     }
 166
 167     /**
 168      * When the JpaDroolsPdpsConnector.standDown() method is invoked, it needs
 169      * access to myPdp, so it can keep its designation status in sync with the
 170      * DB.
 171      *
 172      * @param designated is designated value
 173      */
 174     public static void setMyPdpDesignated(boolean designated) {
 175         logger.debug("setMyPdpDesignated: designated= {}", designated);
 176         myPdp.setDesignated(designated);
 177     }
 178
 179     private class DesignationWaiter extends TimerTask {
 180         // get an instance of logger
 181         private final Logger  logger = LoggerFactory.getLogger(DesignationWaiter.class);
 182
 183         @Override
 184         public void run() {
 185             try {
 186                 logger.debug("DesignatedWaiter.run: Entering");
 187
 188                 //This is for testing the checkWaitTimer
 189                 if (unitTesting && stalled) {
 190                     logger.debug("DesignatedWaiter.run: isUnitTesting = {} isStalled = {}",
 191                                     unitTesting, stalled);
 192                     return;
 193                 }
 194
 195                 synchronized (designationWaiterLock) {
 196
 197                     logger.debug("DesignatedWaiter.run: Entering synchronized block");
 198
 199                     //It is possible that multiple PDPs are designated lead.  So, we will make a list of all designated
 200                     //PDPs and then decide which one really should be designated at the end.
 201                     List<DroolsPdp> listOfDesignated = new ArrayList<>();
 202
 203                     Collection<DroolsPdp> pdps = pdpsConnector.getDroolsPdps();
 204
 205                     logger.debug("DesignatedWaiter.run: pdps.size= {}", pdps.size());
 206
 207                     //This is only true if all designated PDPs have failed
 208                     allPdpsFailed(pdps, listOfDesignated);
 209
 210                     /*
 211                      * We have checked the four combinations of isDesignated and isCurrent.  Where appropriate,
 212                      * we added the PDPs to the potential list of designated pdps
 213                      *
 214                      * We need to give priority to pdps on the same site that is currently being used
 215                      * First, however, we must sanitize the list of designated to make sure their are
 216                      * only designated members or non-designated members.  There should not be both in
 217                      * the list. Because there are real time delays, it is possible that both types could
 218                      * be on the list.
 219                      */
 220
 221                     listOfDesignated = santizeDesignatedList(listOfDesignated);
 222
 223                     /*
 224                      * We need to figure out the last pdp that was the primary so we can get the last site
 225                      * name and the last session numbers.  We need to create a "dummy" droolspdp since
 226                      * it will be used in later comparisons and cannot be null.
 227                      */
 228
 229                     DroolsPdp mostRecentPrimary = computeMostRecentPrimary(pdps, listOfDesignated);
 230
 231                     if (mostRecentPrimary != null) {
 232                         pdpdLastActive = mostRecentPrimary.getPdpId();
 233                     }
 234
 235
 236                     /*
 237                      * It is possible to get here with more than one pdp designated and providing service. This normally
 238                      * occurs when there is a race condition with multiple nodes coming up at the same time. If that is
 239                      * the case we must determine which one is the one that should be designated and which one should
 240                      * be demoted.
 241                      *
 242                      * It is possible to have 0, 1, 2 or more but not all, or all designated.
 243                      *   If we have one designated and current, we chose it and are done
 244                      *   If we have 2 or more, but not all, we must determine which one is in the same site as
 245                      *   the previously designated pdp.
 246                      */
 247                     DroolsPdp designatedPdp = computeDesignatedPdp(listOfDesignated, mostRecentPrimary);
 248
 249                     if (designatedPdp == null) {
 250                         logger.warn("WARNING: DesignatedWaiter.run: No viable PDP found to be Designated. "
 251                             + "designatedPdp still null.");
 252                         designateNoPdp();
 253                         return;
 254                     }
 255
 256                     pdpdNowActive = designatedPdp.getPdpId();
 257
 258                     if (pdpdNowActive.equals(myPdp.getPdpId())) {
 259                         logger.debug("DesignatedWaiter.run: designatedPdp is PDP={}", myPdp.getPdpId());
 260                         designateMyPdp();
 261                         return;
 262                     }
 263
 264                     isDesignated = false;
 265
 266                 } // end synchronized
 267                 logger.debug("DesignatedWaiter.run: myPdp: {}; Returning, isDesignated= {}",
 268                                 isDesignated, myPdp.getPdpId());
 269
 270                 Date tmpDate = currentTime.getDate();
 271                 logger.debug("DesignatedWaiter.run (end of run) waitTimerLastRunDate = {}", tmpDate);
 272
 273                 waitTimerLastRunDate = tmpDate;
 274                 myPdp.setUpdatedDate(waitTimerLastRunDate);
 275                 pdpsConnector.update(myPdp);
 276
 277             } catch (Exception e) {
 278                 logger.error("DesignatedWaiter.run caught an unexpected exception: ", e);
 279             }
 280         } // end run
 281
 282         private void allPdpsFailed(Collection<DroolsPdp> pdps, List<DroolsPdp> listOfDesignated) {
 283             boolean designatedPdpHasFailed = pdpsConnector.hasDesignatedPdpFailed(pdps);
 284             logger.debug("DesignatedWaiter.run: designatedPdpHasFailed= {}", designatedPdpHasFailed);
 285             for (DroolsPdp pdp : pdps) {
 286                 logger.debug("DesignatedWaiter.run: evaluating pdp ID: {}", pdp.getPdpId());
 287
 288                 /*
 289                  * Note: side effect of isPdpCurrent is that any stale but
 290                  * designated PDPs will be marked as un-designated.
 291                  */
 292                 boolean isCurrent = pdpsConnector.isPdpCurrent(pdp);
 293
 294                 /*
 295                  * We can't use stateManagement.getStandbyStatus() here, because
 296                  * we need the standbyStatus, not for this PDP, but for the PDP
 297                  * being processed by this loop iteration.
 298                  */
 299                 String standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 300                 if (standbyStatus == null) {
 301                     // Treat this case as a cold standby -- if we
 302                     // abort here, no sessions will be created in a
 303                     // single-node test environment.
 304                     standbyStatus = StateManagement.COLD_STANDBY;
 305                 }
 306                 logger.debug("DesignatedWaiter.run: PDP= {},  isCurrent= {}", pdp.getPdpId(), isCurrent);
 307
 308                 adjustPdp(pdp, isCurrent, designatedPdpHasFailed, standbyStatus, listOfDesignated);
 309
 310
 311             } // end pdps loop
 312         }
 313
 314         private void adjustPdp(DroolsPdp pdp, boolean isCurrent, boolean designatedPdpHasFailed, String standbyStatus,
 315                         List<DroolsPdp> listOfDesignated) {
 316             /*
 317              * There are 4 combinations of isDesignated and isCurrent.  We will examine each one in-turn
 318              * and evaluate the each pdp in the list of pdps against each combination.
 319              */
 320             if (pdp.isDesignated()) {
 321                 /*
 322                  * This is the first combination of isDesignated and isCurrent
 323                  */
 324                 if (isCurrent) {
 325                     pdpDesignatedCurrent(pdp, standbyStatus, listOfDesignated);
 326
 327                 /*
 328                  * The second combination of isDesignated and isCurrent
 329                  *
 330                  * PDP is designated but not current; it has failed.
 331                  * So we stand it down (it doesn't matter what
 332                  * its standbyStatus is). None of these go on the list.
 333                  */
 334                 } else {
 335                     logger.debug("INFO: DesignatedWaiter.run: PDP= {} is currently "
 336                                     + "designated but is not current; "
 337                                     + "it has failed.  Standing down.  standbyStatus= {}",
 338                                     pdp.getPdpId(), standbyStatus);
 339                     pdpDesignatedNotCurrent(pdp);
 340                 }
 341
 342             } else {
 343                 // NOT designated
 344
 345
 346                 /*
 347                  * The third combination of isDesignated and isCurrent
 348                  * /*
 349                  * If a PDP is not currently designated but is providing service
 350                  * (erroneous, but recoverable) or hot standby
 351                  * we can add it to the list of possible designated if all the designated have failed
 352                  */
 353                 if (isCurrent) {
 354                     pdpNotDesignatedCurrent(pdp, designatedPdpHasFailed, standbyStatus,
 355                                     listOfDesignated);
 356
 357                 /*
 358                  * The fourth combination of isDesignated and isCurrent
 359                  *
 360                  * We are not going to put any of these on the list since it appears they have failed.
 361                  *
 362                  */
 363                 } else {
 364                     logger.debug("INFO: DesignatedWaiter.run: PDP= {} "
 365                                     + "designated= {}, current= {}, "
 366                                     + "designatedPdpHasFailed= {}, "
 367                                     + "standbyStatus= {}", pdp.getPdpId(),
 368                                     pdp.isDesignated(), false, designatedPdpHasFailed, standbyStatus);
 369                     pdpNotDesignatedNotCurrent(pdp, standbyStatus);
 370                 }
 371             }
 372         }
 373
 374         private void pdpDesignatedCurrent(DroolsPdp pdp, String standbyStatus, List<DroolsPdp> listOfDesignated) {
 375             //It is current, but it could have a standbystatus=coldstandby / hotstandby
 376             //If so, we need to stand it down and demote it
 377             if (!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)) {
 378                 if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 379                     logger.debug("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 380                                     + "butstandbystatus is not providingservice. "
 381                                     + " Executing stateManagement.demote()" + "\n\n", myPdp.getPdpId());
 382                     // So, we must demote it
 383                     try {
 384                         demoteMyPdp(pdp, standbyStatus);
 385                     } catch (Exception e) {
 386                         logger.error("DesignatedWaiter.run: myPdp: {} "
 387                                 + "Caught Exception attempting to demote myPdp,"
 388                                 + "message= {}", myPdp.getPdpId(), e);
 389                     }
 390                 } else {
 391                     // Don't demote a remote PDP that is current.  It should catch itself
 392                     logger.debug("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 393                                     + "but standbystatus is not providingservice. "
 394                                     + " Cannot execute stateManagement.demote() "
 395                                     + "since it it is not myPdp\n\n",
 396                                     myPdp.getPdpId());
 397                 }
 398
 399             } else {
 400                 // If we get here, it is ok to be on the list
 401                 logger.debug("DesignatedWaiter.run: PDP= {} is designated, "
 402                                 + "current and {} Noting PDP as "
 403                                 + "designated, standbyStatus= {}",
 404                                 pdp.getPdpId(), standbyStatus, standbyStatus);
 405                 listOfDesignated.add(pdp);
 406             }
 407         }
 408
 409         private void demoteMyPdp(DroolsPdp pdp, String standbyStatus) throws Exception {
 410             /*
 411              * Keep the order like this. StateManagement is last since it triggers
 412              * controller shutdown. This will change isDesignated and it can enter another
 413              * if-combination below
 414              */
 415             pdpsConnector.standDownPdp(pdp.getPdpId());
 416             myPdp.setDesignated(false);
 417             isDesignated = false;
 418             if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 419                     || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 420                 /*
 421                  * Only demote it if it appears it has not already been demoted. Don't worry
 422                  * about synching with the topic endpoint states.  That is done by the
 423                  * refreshStateAudit
 424                  */
 425                 stateManagementFeature.demote();
 426             }
 427         }
 428
 429         private void pdpDesignatedNotCurrent(DroolsPdp pdp) {
 430             /*
 431              * Changes designated to 0 but it is still potentially providing service.
 432              * Will affect isDesignated, so, it can enter an if-combination below
 433              */
 434             pdpsConnector.standDownPdp(pdp.getPdpId());
 435
 436             //need to change standbystatus to coldstandby
 437             if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 438                 logger.debug("\n\nDesignatedWaiter.run: myPdp {} is not Current. "
 439                                 + " Executing stateManagement.disableFailed()\n\n", myPdp.getPdpId());
 440                 // We found that myPdp is designated but not current
 441                 // So, we must cause it to disableFail
 442                 try {
 443                     myPdp.setDesignated(false);
 444                     pdpsConnector.setDesignated(myPdp, false);
 445                     isDesignated = false;
 446                     stateManagementFeature.disableFailed();
 447                 } catch (Exception e) {
 448                     logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception "
 449                             + "attempting to disableFail myPdp {}, message= {}",
 450                             myPdp.getPdpId(), myPdp.getPdpId(), e);
 451                 }
 452             } else { //it is a remote PDP that is failed
 453                 logger.debug("\n\nDesignatedWaiter.run: PDP {} is not Current. "
 454                                 + " Executing stateManagement.disableFailed(otherResourceName)\n\n",
 455                                 pdp.getPdpId());
 456                 // We found a PDP is designated but not current
 457                 // We already called standdown(pdp) which will change designated to false
 458                 // Now we need to disableFail it to get its states in synch.  The standbyStatus
 459                 // should equal coldstandby
 460                 try {
 461                     stateManagementFeature.disableFailed(pdp.getPdpId());
 462                 } catch (Exception e) {
 463                     logger.error("DesignatedWaiter.run: for PDP {}  Caught Exception attempting to "
 464                             + "disableFail({}), message= {}",
 465                             pdp.getPdpId(), pdp.getPdpId(), e);
 466                 }
 467
 468             }
 469         }
 470
 471         private void pdpNotDesignatedCurrent(DroolsPdp pdp, boolean designatedPdpHasFailed, String standbyStatus,
 472                         List<DroolsPdp> listOfDesignated) {
 473             if (!(StateManagement.HOT_STANDBY.equals(standbyStatus)
 474                     || StateManagement.COLD_STANDBY.equals(standbyStatus))) {
 475                 logger.debug("\n\nDesignatedWaiter.run: PDP {}"
 476                                 + " is NOT designated but IS current and"
 477                                 + " has a standbystatus= {}", pdp.getPdpId(), standbyStatus);
 478                 // Since it is current, we assume it can adjust its own state.
 479                 // We will demote if it is myPdp
 480                 if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 481                     //demote it
 482                     logger.debug("DesignatedWaiter.run: PDP {} going to "
 483                                     + "setDesignated = false and calling stateManagement.demote",
 484                                     pdp.getPdpId());
 485                     try {
 486                         //Keep the order like this.
 487                         //StateManagement is last since it triggers controller shutdown
 488                         pdpsConnector.setDesignated(myPdp, false);
 489                         myPdp.setDesignated(false);
 490                         isDesignated = false;
 491                         //This is definitely not a redundant call.
 492                         //It is attempting to correct a problem
 493                         stateManagementFeature.demote();
 494                         //recheck the standbystatus
 495                         standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 496                     } catch (Exception e) {
 497                         logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception "
 498                                 + "attempting to demote myPdp {}, message = {}",  myPdp.getPdpId(),
 499                                 myPdp.getPdpId(), e);
 500                     }
 501
 502                 }
 503             }
 504             if (StateManagement.HOT_STANDBY.equals(standbyStatus) && designatedPdpHasFailed) {
 505                 //add it to the list
 506                 logger.debug("INFO: DesignatedWaiter.run: PDP= {}"
 507                                 + " is not designated but is {} and designated PDP "
 508                                 + "has failed.  standbyStatus= {}", pdp.getPdpId(),
 509                                 standbyStatus, standbyStatus);
 510                 listOfDesignated.add(pdp);
 511             }
 512         }
 513
 514         private void pdpNotDesignatedNotCurrent(DroolsPdp pdp, String standbyStatus) {
 515             if (StateManagement.COLD_STANDBY.equals(standbyStatus)) {
 516                 return;
 517             }
 518
 519             //stand it down
 520             //disableFail it
 521             pdpsConnector.standDownPdp(pdp.getPdpId());
 522             if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 523                 /*
 524                  * I don't actually know how this condition could
 525                  * happen, but if it did, we would want to declare it
 526                  * failed.
 527                  */
 528                 logger.debug("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 529                                 + " Executing stateManagement.disableFailed()\n\n",
 530                                 myPdp.getPdpId());
 531                 // So, we must disableFail it
 532                 try {
 533                     //Keep the order like this.
 534                     //StateManagement is last since it triggers controller shutdown
 535                     pdpsConnector.setDesignated(myPdp, false);
 536                     myPdp.setDesignated(false);
 537                     isDesignated = false;
 538                     stateManagementFeature.disableFailed();
 539                 } catch (Exception e) {
 540                     logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 541                             + "disableFail myPdp {}, message= {}",
 542                             myPdp.getPdpId(), myPdp.getPdpId(), e);
 543                 }
 544             } else { //it is remote
 545                 logger.debug("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 546                                 + " Executing stateManagement.disableFailed({})\n\n",
 547                                 myPdp.getPdpId(), pdp.getPdpId());
 548                 // We already called standdown(pdp) which will change designated to false
 549                 // Now we need to disableFail it to get its states in sync.
 550                 // StandbyStatus = coldstandby
 551                 try {
 552                     stateManagementFeature.disableFailed(pdp.getPdpId());
 553                 } catch (Exception e) {
 554                     logger.error("DesignatedWaiter.run: for PDP {}"
 555                             + " Caught Exception attempting to disableFail({})"
 556                             + ", message=", pdp.getPdpId(), pdp.getPdpId(), e);
 557                 }
 558             }
 559         }
 560
 561         private void designateNoPdp() {
 562             // Just to be sure the parameters are correctly set
 563             myPdp.setDesignated(false);
 564             pdpsConnector.setDesignated(myPdp, false);
 565             isDesignated = false;
 566
 567             waitTimerLastRunDate = currentTime.getDate();
 568             logger.debug("DesignatedWaiter.run (designatedPdp == null) waitTimerLastRunDate = {}",
 569                             waitTimerLastRunDate);
 570             myPdp.setUpdatedDate(waitTimerLastRunDate);
 571             pdpsConnector.update(myPdp);
 572         }
 573
 574         private void designateMyPdp() {
 575             /*
 576              * update function expects myPdp.isDesignated to be true.
 577              */
 578             try {
 579                 //Keep the order like this.  StateManagement is last since it triggers controller init
 580                 myPdp.setDesignated(true);
 581                 myPdp.setDesignatedDate(currentTime.getDate());
 582                 pdpsConnector.setDesignated(myPdp, true);
 583                 isDesignated = true;
 584                 String standbyStatus = stateManagementFeature.getStandbyStatus();
 585                 if (!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)) {
 586                     /*
 587                      * Only call promote if it is not already in the right state.  Don't worry about
 588                      * synching the lower level topic endpoint states.  That is done by the
 589                      * refreshStateAudit.
 590                      */
 591                     stateManagementFeature.promote();
 592                 }
 593             } catch (Exception e) {
 594                 logger.error("ERROR: DesignatedWaiter.run: Caught Exception attempting to promote PDP={}"
 595                         + ", message=", myPdp.getPdpId(), e);
 596                 myPdp.setDesignated(false);
 597                 pdpsConnector.setDesignated(myPdp, false);
 598                 isDesignated = false;
 599                 //If you can't promote it, demote it
 600                 try {
 601                     String standbyStatus = stateManagementFeature.getStandbyStatus();
 602                     if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 603                             || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 604                         /*
 605                          * Only call demote if it is not already in the right state.  Don't worry about
 606                          * synching the lower level topic endpoint states.  That is done by the
 607                          * refreshStateAudit.
 608                          */
 609                         stateManagementFeature.demote();
 610                     }
 611                 } catch (Exception e1) {
 612                     logger.error("ERROR: DesignatedWaiter.run: Caught StandbyStatusException "
 613                             + "attempting to promote then demote PDP={}, message=",
 614                             myPdp.getPdpId(), e1);
 615                 }
 616
 617             }
 618             waitTimerLastRunDate = currentTime.getDate();
 619             logger.debug("DesignatedWaiter.run (designatedPdp.getPdpId().equals(myPdp.getPdpId())) "
 620                             + "waitTimerLastRunDate = {}", waitTimerLastRunDate);
 621             myPdp.setUpdatedDate(waitTimerLastRunDate);
 622             pdpsConnector.update(myPdp);
 623         }
 624     }
 625
 626     /**
 627      * Sanitize designated list.
 628      *
 629      * @param listOfDesignated list of designated pdps
 630      * @return list of drools pdps
 631      */
 632     public List<DroolsPdp> santizeDesignatedList(List<DroolsPdp> listOfDesignated) {
 633
 634         boolean containsDesignated = false;
 635         boolean containsHotStandby = false;
 636         List<DroolsPdp> listForRemoval = new ArrayList<>();
 637         for (DroolsPdp pdp : listOfDesignated) {
 638             logger.debug("DesignatedWaiter.run sanitizing: pdp = {}"
 639                             + " isDesignated = {}", pdp.getPdpId(), pdp.isDesignated());
 640             if (pdp.isDesignated()) {
 641                 containsDesignated = true;
 642             } else {
 643                 containsHotStandby = true;
 644                 listForRemoval.add(pdp);
 645             }
 646         }
 647         if (containsDesignated && containsHotStandby) {
 648             //remove the hot standby from the list
 649             listOfDesignated.removeAll(listForRemoval);
 650         }
 651         return listOfDesignated;
 652     }
 653
 654     /**
 655      * Compute most recent primary.
 656      *
 657      * @param pdps collection of pdps
 658      * @param listOfDesignated list of designated pdps
 659      * @return drools pdp object
 660      */
 661     public DroolsPdp computeMostRecentPrimary(Collection<DroolsPdp> pdps, List<DroolsPdp> listOfDesignated) {
 662         boolean containsDesignated = listOfDesignated.stream().anyMatch(DroolsPdp::isDesignated);
 663
 664         DroolsPdp mostRecentPrimary = new DroolsPdpImpl(null, true, 1, new Date(0));
 665         mostRecentPrimary.setSite(null);
 666         logger.debug("DesignatedWaiter.run listOfDesignated.size() = {}", listOfDesignated.size());
 667
 668         if (listOfDesignated.size() <= 1) {
 669             logger.debug("DesignatedWainter.run: listOfDesignated.size <=1");
 670             //Only one or none is designated or hot standby.  Choose the latest designated date
 671             mostRecentPrimary = getLatestDesignated(pdps, mostRecentPrimary);
 672
 673         } else if (listOfDesignated.size() == pdps.size()) {
 674             logger.debug("DesignatedWainter.run: listOfDesignated.size = pdps.size() which is {}", pdps.size());
 675             //They are all designated or all hot standby.
 676             mostRecentPrimary = getBestDesignated(pdps, containsDesignated);
 677
 678         } else {
 679             logger.debug("DesignatedWainter.run: Some but not all are designated or hot standby. ");
 680             logger.debug("DesignatedWainter.run: containsDesignated = {}", containsDesignated);
 681             //Some but not all are designated or hot standby.
 682             if (containsDesignated) {
 683                 /*
 684                  * The list only contains designated.  This is a problem.  It is most likely a race
 685                  * condition that resulted in two thinking they should be designated. Choose the
 686                  * site with the latest designated date for the pdp not included on the designated list.
 687                  * This should be the site that had the last designation before this race condition
 688                  * occurred.
 689                  */
 690                 mostRecentPrimary = getLatestUndesignated(pdps, mostRecentPrimary, listOfDesignated);
 691
 692             } else {
 693                 //The list only contains hot standby. Choose the site of the latest designated date
 694                 mostRecentPrimary = getLatestDesignated(pdps, mostRecentPrimary);
 695             }
 696         }
 697         return mostRecentPrimary;
 698     }
 699
 700     private DroolsPdp getBestDesignated(Collection<DroolsPdp> pdps, boolean containsDesignated) {
 701         DroolsPdp mostRecentPrimary;
 702         mostRecentPrimary = null;
 703         for (DroolsPdp pdp : pdps) {
 704             if (mostRecentPrimary == null) {
 705                 mostRecentPrimary = pdp;
 706                 continue;
 707             }
 708             if (containsDesignated) { //Choose the site of the first designated date
 709                 if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) < 0) {
 710                     mostRecentPrimary = pdp;
 711                     logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 712                 }
 713             } else { //Choose the site with the latest designated date
 714                 if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 715                     mostRecentPrimary = pdp;
 716                     logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 717                 }
 718             }
 719         }
 720         return mostRecentPrimary;
 721     }
 722
 723     private DroolsPdp getLatestUndesignated(Collection<DroolsPdp> pdps, DroolsPdp mostRecentPrimary,
 724                     List<DroolsPdp> listOfDesignated) {
 725         for (DroolsPdp pdp : pdps) {
 726             if (listOfDesignated.contains(pdp)) {
 727                 continue; //Don't consider this entry
 728             }
 729             if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 730                 mostRecentPrimary = pdp;
 731                 logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 732             }
 733         }
 734         return mostRecentPrimary;
 735     }
 736
 737     private DroolsPdp getLatestDesignated(Collection<DroolsPdp> pdps, DroolsPdp mostRecentPrimary) {
 738         for (DroolsPdp pdp : pdps) {
 739             logger.debug("DesignatedWaiter.run pdp = {}"
 740                             + " pdp.getDesignatedDate() = {}",
 741                             pdp.getPdpId(), pdp.getDesignatedDate());
 742             if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 743                 mostRecentPrimary = pdp;
 744                 logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 745             }
 746         }
 747         return mostRecentPrimary;
 748     }
 749
 750     /**
 751      * Compue designated pdp.
 752      *
 753      * @param listOfDesignated list of designated pdps
 754      * @param mostRecentPrimary most recent primary pdpd
 755      * @return drools pdp object
 756      */
 757     public DroolsPdp computeDesignatedPdp(List<DroolsPdp> listOfDesignated, DroolsPdp mostRecentPrimary) {
 758         if (listOfDesignated.isEmpty()) {
 759             logger.debug("\nDesignatedWaiter.run: myPdp: {} listOfDesignated is: EMPTY.", myPdp.getPdpId());
 760             return null;
 761         }
 762
 763         if (listOfDesignated.size() == 1) {
 764             logger.debug("\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 765                             + "has ONE entry. PDP ID: {}", myPdp.getPdpId(), listOfDesignated.get(0).getPdpId());
 766             return listOfDesignated.get(0);
 767         }
 768
 769         logger.debug("DesignatedWaiter.run: myPdp: {} listOfDesignated.size(): {}", myPdp.getPdpId(),
 770                         listOfDesignated.size());
 771         DesignatedData data = new DesignatedData();
 772         for (DroolsPdp pdp : listOfDesignated) {
 773             DroolsPdp rejectedPdp;
 774
 775             // We need to determine if another PDP is the lowest priority
 776             if (Objects.equals(pdp.getSite(), mostRecentPrimary.getSite())) {
 777                 rejectedPdp = data.compareSameSite(pdp);
 778             } else {
 779                 rejectedPdp = data.compareDifferentSite(pdp);
 780             }
 781             // If the rejectedPdp is myPdp, we need to stand it down and demote it.  Each pdp is responsible
 782             // for demoting itself
 783             if (rejectedPdp != null && Objects.equals(rejectedPdp.getPdpId(), myPdp.getPdpId())) {
 784                 logger.debug("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated myPdp ID: {}"
 785                                 + " is NOT the lowest priority.  Executing stateManagement.demote()\n\n",
 786                                 myPdp.getPdpId(),
 787                                 myPdp.getPdpId());
 788                 // We found that myPdp is on the listOfDesignated and it is not the lowest priority
 789                 // So, we must demote it
 790                 demoteMyPdp();
 791             }
 792         }
 793
 794         DroolsPdp lowestPriorityPdp = data.getLowestPriority();
 795
 796         //now we have a valid value for lowestPriorityPdp
 797         logger.debug("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 798                         + "found the LOWEST priority pdp ID: {} "
 799                         + " It is now the designatedPpd from the perspective of myPdp ID: {} \n\n",
 800                         myPdp.getPdpId(), lowestPriorityPdp.getPdpId(), myPdp);
 801         return lowestPriorityPdp;
 802
 803     }
 804
 805     private class DesignatedData {
 806         private DroolsPdp lowestPrioritySameSite = null;
 807         private DroolsPdp lowestPriorityDifferentSite = null;
 808
 809         private DroolsPdp compareSameSite(DroolsPdp pdp) {
 810             if (lowestPrioritySameSite == null) {
 811                 if (lowestPriorityDifferentSite != null) {
 812                     //we need to reject lowestPriorityDifferentSite
 813                     DroolsPdp rejectedPdp = lowestPriorityDifferentSite;
 814                     lowestPriorityDifferentSite = pdp;
 815                     return rejectedPdp;
 816                 }
 817                 lowestPrioritySameSite = pdp;
 818                 return null;
 819             } else {
 820                 if (pdp.getPdpId().equals((lowestPrioritySameSite.getPdpId()))) {
 821                     return null;    //nothing to compare
 822                 }
 823                 if (pdp.comparePriority(lowestPrioritySameSite) < 0) {
 824                     logger.debug("\nDesignatedWaiter.run: myPdp {}  listOfDesignated pdp ID: {}"
 825                                     + " has lower priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 826                                     lowestPrioritySameSite.getPdpId());
 827                     //we need to reject lowestPrioritySameSite
 828                     DroolsPdp rejectedPdp = lowestPrioritySameSite;
 829                     lowestPrioritySameSite = pdp;
 830                     return rejectedPdp;
 831                 } else {
 832                     //we need to reject pdp and keep lowestPrioritySameSite
 833                     logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {} "
 834                                     + " has higher priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 835                                     lowestPrioritySameSite.getPdpId());
 836                     return pdp;
 837                 }
 838             }
 839         }
 840
 841         private DroolsPdp compareDifferentSite(DroolsPdp pdp) {
 842             if (lowestPrioritySameSite != null) {
 843                 //if we already have a candidate for same site, we don't want to bother with different sites
 844                 return pdp;
 845             } else {
 846                 if (lowestPriorityDifferentSite == null) {
 847                     lowestPriorityDifferentSite = pdp;
 848                     return null;
 849                 }
 850                 if (pdp.getPdpId().equals((lowestPriorityDifferentSite.getPdpId()))) {
 851                     return null;    //nothing to compare
 852                 }
 853                 if (pdp.comparePriority(lowestPriorityDifferentSite) < 0) {
 854                     logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 855                                     + " has lower priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 856                                     lowestPriorityDifferentSite.getPdpId());
 857                     //we need to reject lowestPriorityDifferentSite
 858                     DroolsPdp rejectedPdp = lowestPriorityDifferentSite;
 859                     lowestPriorityDifferentSite = pdp;
 860                     return rejectedPdp;
 861                 } else {
 862                     //we need to reject pdp and keep lowestPriorityDifferentSite
 863                     logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 864                                     + " has higher priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 865                                     lowestPriorityDifferentSite.getPdpId());
 866                     return pdp;
 867                 }
 868             }
 869         }
 870
 871         private DroolsPdp getLowestPriority() {
 872             return (lowestPrioritySameSite != null ? lowestPrioritySameSite : lowestPriorityDifferentSite);
 873         }
 874     }
 875
 876     private void demoteMyPdp() {
 877         try {
 878             //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 879             myPdp.setDesignated(false);
 880             pdpsConnector.setDesignated(myPdp, false);
 881             isDesignated = false;
 882             String standbyStatus = stateManagementFeature.getStandbyStatus();
 883             if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 884                     || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 885                 /*
 886                  * Only call demote if it is not already in the right state.  Don't worry about
 887                  * synching the lower level topic endpoint states.  That is done by the
 888                  * refreshStateAudit.
 889                  */
 890                 stateManagementFeature.demote();
 891             }
 892         } catch (Exception e) {
 893             myPdp.setDesignated(false);
 894             pdpsConnector.setDesignated(myPdp, false);
 895             isDesignated = false;
 896             logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 897                     + "demote myPdp {} myPdp.getPdpId(), message= {}", myPdp.getPdpId(),
 898                     e);
 899         }
 900     }
 901
 902     private class TimerUpdateClass extends TimerTask {
 903
 904         @Override
 905         public void run() {
 906             try {
 907                 logger.debug("TimerUpdateClass.run: entry");
 908                 checkWaitTimer();
 909             } catch (Exception e) {
 910                 logger.error("TimerUpdateClass.run caught an unexpected exception: ", e);
 911             }
 912             logger.debug("TimerUpdateClass.run.exit");
 913         }
 914     }
 915
 916     @Override
 917     public void checkThreadStatus() {
 918         checkWaitTimer();
 919     }
 920
 921     private void checkWaitTimer() {
 922         synchronized (checkWaitTimerLock) {
 923             try {
 924                 logger.debug("checkWaitTimer: entry");
 925                 Date now = currentTime.getDate();
 926                 long nowMs = now.getTime();
 927                 long waitTimerMs = waitTimerLastRunDate.getTime();
 928
 929                 //give it 10 times leeway
 930                 if ((nowMs - waitTimerMs)  > 10 * pdpUpdateInterval) {
 931                     if (allSeemsWell == null || allSeemsWell) {
 932                         allSeemsWell = false;
 933                         logger.debug("checkWaitTimer: calling allSeemsWell with ALLNOTWELL param");
 934                         stateManagementFeature.allSeemsWell(this.getClass().getName(),
 935                                 StateManagementFeatureApiConstants.ALLNOTWELL_STATE,
 936                                 "DesignationWaiter/ElectionHandler has STALLED");
 937                     }
 938                     logger.error("checkWaitTimer: nowMs - waitTimerMs = {}"
 939                             + ", exceeds 10* pdpUpdateInterval = {}"
 940                             + " DesignationWaiter is STALLED!", (nowMs - waitTimerMs), (10 * pdpUpdateInterval));
 941                 } else if (allSeemsWell == null || !allSeemsWell) {
 942                     allSeemsWell = true;
 943                     stateManagementFeature.allSeemsWell(this.getClass().getName(),
 944                             StateManagementFeatureApiConstants.ALLSEEMSWELL_STATE,
 945                             "DesignationWaiter/ElectionHandler has RESUMED");
 946                     logger.info("DesignationWaiter/ElectionHandler has RESUMED");
 947                 }
 948                 logger.debug("checkWaitTimer: exit");
 949             } catch (Exception e) {
 950                 logger.error("checkWaitTimer: caught unexpected exception: ", e);
 951             }
 952         }
 953     }
 954
 955     private long getDWaiterStartMs() {
 956         Date now = currentTime.getDate();
 957
 958         // Retrieve the ms since the epoch
 959         long nowMs = now.getTime();
 960
 961         // Time since the end of the last pdpUpdateInterval multiple
 962         long nowModMs = nowMs % pdpUpdateInterval;
 963
 964         // Time to the start of the next pdpUpdateInterval multiple
 965         long startMs = 2 * pdpUpdateInterval - nowModMs;
 966
 967         // Give the start time a minimum of a 5 second cushion
 968         if (startMs < 5000) {
 969             // Start at the beginning  of following interval
 970             startMs = pdpUpdateInterval + startMs;
 971         }
 972         return startMs;
 973     }
 974 }