gerrit.onap Code Review - policy/drools-pdp.git/blob

   1 /*
   2  * ============LICENSE_START=======================================================
   3  * feature-active-standby-management
   4  * ================================================================================
   5  * Copyright (C) 2017-2020 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.policy.drools.activestandby;
  22
  23 import java.util.ArrayList;
  24 import java.util.Collection;
  25 import java.util.Date;
  26 import java.util.List;
  27 import java.util.Timer;
  28 import java.util.TimerTask;
  29 import org.onap.policy.common.im.MonitorTime;
  30 import org.onap.policy.common.im.StateManagement;
  31 import org.onap.policy.common.utils.time.CurrentTime;
  32 import org.onap.policy.drools.statemanagement.StateManagementFeatureApi;
  33 import org.onap.policy.drools.statemanagement.StateManagementFeatureApiConstants;
  34 import org.slf4j.Logger;
  35 import org.slf4j.LoggerFactory;
  36
  37 public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
  38     private static final String RUN_PRIMARY_MSG = "DesignatedWaiter.run mostRecentPrimary = {}";
  39
  40     // get an instance of logger
  41     private static final Logger  logger = LoggerFactory.getLogger(DroolsPdpsElectionHandler.class);
  42     private DroolsPdpsConnector pdpsConnector;
  43     private Object checkWaitTimerLock = new Object();
  44     private Object designationWaiterLock = new Object();
  45
  46     /*
  47      * Must be static, so it can be referenced by JpaDroolsPdpsConnector,
  48      * without requiring a reference to the election handler instantiation.
  49      */
  50     private static DroolsPdp myPdp;
  51
  52     private Date waitTimerLastRunDate;
  53
  54     // The interval between runs of the DesignationWaiter
  55     private int pdpUpdateInterval;
  56
  57     private volatile boolean isDesignated;
  58
  59     private String pdpdNowActive;
  60     private String pdpdLastActive;
  61
  62     /*
  63      * Start allSeemsWell with a value of null so that, on the first run
  64      * of the checkWaitTimer it will set the value in IntegrityMonitor
  65      * regardless of whether it needs to be set to true or false.
  66      */
  67     private Boolean allSeemsWell = null;
  68
  69     private StateManagementFeatureApi stateManagementFeature;
  70
  71     private final CurrentTime currentTime = MonitorTime.getInstance();
  72
  73     private static boolean isUnitTesting = false;
  74     private static boolean isStalled = false;
  75
  76     /**
  77      * Constructor.
  78      *
  79      * @param pdps connectors
  80      * @param myPdp pdp
  81      */
  82     public DroolsPdpsElectionHandler(DroolsPdpsConnector pdps, DroolsPdp myPdp) {
  83         if (pdps == null) {
  84             logger.error("DroolsPdpsElectinHandler(): pdpsConnector==null");
  85             throw new IllegalArgumentException("DroolsPdpsElectinHandler(): pdpsConnector==null");
  86         }
  87         if (myPdp == null) {
  88             logger.error("DroolsPdpsElectinHandler(): droolsPdp==null");
  89             throw new IllegalArgumentException("DroolsPdpsElectinHandler(): DroolsPdp==null");
  90         }
  91
  92         pdpdNowActive = null;
  93         pdpdLastActive = null;
  94         this.pdpsConnector = pdps;
  95         setMyPdp(myPdp);
  96         this.isDesignated = false;
  97
  98         // The interval between checks of the DesignationWaiter to be sure it is running.
  99         int pdpCheckInterval = 3000;
 100         try {
 101             pdpCheckInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(
 102                     ActiveStandbyProperties.PDP_CHECK_INVERVAL));
 103         } catch (Exception e) {
 104             logger.error("Could not get pdpCheckInterval property. Using default {}", pdpCheckInterval, e);
 105         }
 106         pdpUpdateInterval = 2000;
 107         try {
 108             pdpUpdateInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(
 109                     ActiveStandbyProperties.PDP_UPDATE_INTERVAL));
 110         } catch (Exception e) {
 111             logger.error("Could not get pdpUpdateInterval property. Using default {} ", pdpUpdateInterval, e);
 112         }
 113
 114         Date now = currentTime.getDate();
 115
 116         // Retrieve the ms since the epoch
 117         final long nowMs = now.getTime();
 118
 119         // Create the timer which will update the updateDate in DroolsPdpEntity table.
 120         // This is the heartbeat
 121         Timer updateWorker = Factory.getInstance().makeTimer();
 122
 123         // Schedule the TimerUpdateClass to run at 100 ms and run at pdpCheckInterval ms thereafter
 124         // NOTE: The first run of the TimerUpdateClass results in myPdp being added to the
 125         // drools droolsPdpEntity table.
 126         updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
 127
 128         // Create the timer which will run the election algorithm
 129         Timer waitTimer = Factory.getInstance().makeTimer();
 130
 131         // Schedule it to start in startMs ms
 132         // (so it will run after the updateWorker and run at pdpUpdateInterval ms thereafter
 133         long startMs = getDWaiterStartMs();
 134         DesignationWaiter designationWaiter = new DesignationWaiter();
 135         waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
 136         waitTimerLastRunDate = new Date(nowMs + startMs);
 137
 138         //Get the StateManagementFeature instance
 139
 140         for (StateManagementFeatureApi feature : StateManagementFeatureApiConstants.getImpl().getList()) {
 141             if (feature.getResourceName().equals(myPdp.getPdpId())) {
 142                 logger.debug("DroolsPdpsElectionHandler: Found StateManagementFeature"
 143                                 + " with resourceName: {}", myPdp.getPdpId());
 144                 stateManagementFeature = feature;
 145                 break;
 146             }
 147         }
 148         if (stateManagementFeature == null) {
 149             logger.error("DroolsPdpsElectionHandler failed to initialize.  "
 150                     + "Unable to get instance of StateManagementFeatureApi "
 151                     + "with resourceID: {}", myPdp.getPdpId());
 152         }
 153     }
 154
 155     private static void setMyPdp(DroolsPdp myPdp) {
 156         DroolsPdpsElectionHandler.myPdp = myPdp;
 157     }
 158
 159     public static void setIsUnitTesting(boolean val) {
 160         isUnitTesting = val;
 161     }
 162
 163     public static void setIsStalled(boolean val) {
 164         isStalled = val;
 165     }
 166
 167     /**
 168      * When the JpaDroolsPdpsConnector.standDown() method is invoked, it needs
 169      * access to myPdp, so it can keep its designation status in sync with the
 170      * DB.
 171      *
 172      * @param designated is designated value
 173      */
 174     public static void setMyPdpDesignated(boolean designated) {
 175         logger.debug("setMyPdpDesignated: designated= {}", designated);
 176         myPdp.setDesignated(designated);
 177     }
 178
 179     private class DesignationWaiter extends TimerTask {
 180         // get an instance of logger
 181         private final Logger  logger = LoggerFactory.getLogger(DesignationWaiter.class);
 182
 183         @Override
 184         public void run() {
 185             try {
 186                 logger.debug("DesignatedWaiter.run: Entering");
 187
 188                 //This is for testing the checkWaitTimer
 189                 if (isUnitTesting && isStalled) {
 190                     logger.debug("DesignatedWaiter.run: isUnitTesting = {} isStalled = {}",
 191                                     isUnitTesting, isStalled);
 192                     return;
 193                 }
 194
 195                 synchronized (designationWaiterLock) {
 196
 197                     logger.debug("DesignatedWaiter.run: Entering synchronized block");
 198
 199                     //It is possible that multiple PDPs are designated lead.  So, we will make a list of all designated
 200                     //PDPs and then decide which one really should be designated at the end.
 201                     List<DroolsPdp> listOfDesignated = new ArrayList<>();
 202
 203                     Collection<DroolsPdp> pdps = pdpsConnector.getDroolsPdps();
 204
 205                     logger.debug("DesignatedWaiter.run: pdps.size= {}", pdps.size());
 206
 207                     //This is only true if all designated PDPs have failed
 208                     allPdpsFailed(pdps, listOfDesignated);
 209
 210                     /*
 211                      * We have checked the four combinations of isDesignated and isCurrent.  Where appropriate,
 212                      * we added the PDPs to the potential list of designated pdps
 213                      *
 214                      * We need to give priority to pdps on the same site that is currently being used
 215                      * First, however, we must sanitize the list of designated to make sure their are
 216                      * only designated members or non-designated members.  There should not be both in
 217                      * the list. Because there are real time delays, it is possible that both types could
 218                      * be on the list.
 219                      */
 220
 221                     listOfDesignated = santizeDesignatedList(listOfDesignated);
 222
 223                     /*
 224                      * We need to figure out the last pdp that was the primary so we can get the last site
 225                      * name and the last session numbers.  We need to create a "dummy" droolspdp since
 226                      * it will be used in later comparisons and cannot be null.
 227                      */
 228
 229                     DroolsPdp mostRecentPrimary = computeMostRecentPrimary(pdps, listOfDesignated);
 230
 231                     if (mostRecentPrimary != null) {
 232                         pdpdLastActive = mostRecentPrimary.getPdpId();
 233                     }
 234
 235
 236                     /*
 237                      * It is possible to get here with more than one pdp designated and providing service. This normally
 238                      * occurs when there is a race condition with multiple nodes coming up at the same time. If that is
 239                      * the case we must determine which one is the one that should be designated and which one should
 240                      * be demoted.
 241                      *
 242                      * It is possible to have 0, 1, 2 or more but not all, or all designated.
 243                      *   If we have one designated and current, we chose it and are done
 244                      *   If we have 2 or more, but not all, we must determine which one is in the same site as
 245                      *   the previously designated pdp.
 246                      */
 247                     DroolsPdp designatedPdp = computeDesignatedPdp(listOfDesignated, mostRecentPrimary);
 248
 249                     if (designatedPdp == null) {
 250                         logger.warn("WARNING: DesignatedWaiter.run: No viable PDP found to be Designated. "
 251                             + "designatedPdp still null.");
 252                         designateNoPdp();
 253                         return;
 254                     }
 255
 256                     pdpdNowActive = designatedPdp.getPdpId();
 257
 258                     if (pdpdNowActive.equals(myPdp.getPdpId())) {
 259                         logger.debug("DesignatedWaiter.run: designatedPdp is PDP={}", myPdp.getPdpId());
 260                         designateMyPdp();
 261                         return;
 262                     }
 263
 264                     isDesignated = false;
 265
 266                 } // end synchronized
 267                 logger.debug("DesignatedWaiter.run: myPdp: {}; Returning, isDesignated= {}",
 268                                 isDesignated, myPdp.getPdpId());
 269
 270                 Date tmpDate = currentTime.getDate();
 271                 logger.debug("DesignatedWaiter.run (end of run) waitTimerLastRunDate = {}", tmpDate);
 272
 273                 waitTimerLastRunDate = tmpDate;
 274                 myPdp.setUpdatedDate(waitTimerLastRunDate);
 275                 pdpsConnector.update(myPdp);
 276
 277             } catch (Exception e) {
 278                 logger.error("DesignatedWaiter.run caught an unexpected exception: ", e);
 279             }
 280         } // end run
 281
 282         private void allPdpsFailed(Collection<DroolsPdp> pdps, List<DroolsPdp> listOfDesignated) {
 283             boolean designatedPdpHasFailed = pdpsConnector.hasDesignatedPdpFailed(pdps);
 284             logger.debug("DesignatedWaiter.run: designatedPdpHasFailed= {}", designatedPdpHasFailed);
 285             for (DroolsPdp pdp : pdps) {
 286                 logger.debug("DesignatedWaiter.run: evaluating pdp ID: {}", pdp.getPdpId());
 287
 288                 /*
 289                  * Note: side effect of isPdpCurrent is that any stale but
 290                  * designated PDPs will be marked as un-designated.
 291                  */
 292                 boolean isCurrent = pdpsConnector.isPdpCurrent(pdp);
 293
 294                 /*
 295                  * We can't use stateManagement.getStandbyStatus() here, because
 296                  * we need the standbyStatus, not for this PDP, but for the PDP
 297                  * being processed by this loop iteration.
 298                  */
 299                 String standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 300                 if (standbyStatus == null) {
 301                     // Treat this case as a cold standby -- if we
 302                     // abort here, no sessions will be created in a
 303                     // single-node test environment.
 304                     standbyStatus = StateManagement.COLD_STANDBY;
 305                 }
 306                 logger.debug("DesignatedWaiter.run: PDP= {},  isCurrent= {}", pdp.getPdpId(), isCurrent);
 307
 308                 adjustPdp(pdp, isCurrent, designatedPdpHasFailed, standbyStatus, listOfDesignated);
 309
 310
 311             } // end pdps loop
 312         }
 313
 314         private void adjustPdp(DroolsPdp pdp, boolean isCurrent, boolean designatedPdpHasFailed, String standbyStatus,
 315                         List<DroolsPdp> listOfDesignated) {
 316             /*
 317              * There are 4 combinations of isDesignated and isCurrent.  We will examine each one in-turn
 318              * and evaluate the each pdp in the list of pdps against each combination.
 319              */
 320             if (pdp.isDesignated()) {
 321                 /*
 322                  * This is the first combination of isDesignated and isCurrent
 323                  */
 324                 if (isCurrent) {
 325                     pdpDesignatedCurrent(pdp, standbyStatus, listOfDesignated);
 326
 327                 /*
 328                  * The second combination of isDesignated and isCurrent
 329                  *
 330                  * PDP is designated but not current; it has failed.
 331                  * So we stand it down (it doesn't matter what
 332                  * its standbyStatus is). None of these go on the list.
 333                  */
 334                 } else {
 335                     logger.debug("INFO: DesignatedWaiter.run: PDP= {} is currently "
 336                                     + "designated but is not current; "
 337                                     + "it has failed.  Standing down.  standbyStatus= {}",
 338                                     pdp.getPdpId(), standbyStatus);
 339                     pdpDesignatedNotCurrent(pdp);
 340                 }
 341
 342             } else {
 343                 // NOT designated
 344
 345
 346                 /*
 347                  * The third combination of isDesignated and isCurrent
 348                  * /*
 349                  * If a PDP is not currently designated but is providing service
 350                  * (erroneous, but recoverable) or hot standby
 351                  * we can add it to the list of possible designated if all the designated have failed
 352                  */
 353                 if (isCurrent) {
 354                     pdpNotDesignatedCurrent(pdp, designatedPdpHasFailed, standbyStatus,
 355                                     listOfDesignated);
 356
 357                 /*
 358                  * The fourth combination of isDesignated and isCurrent
 359                  *
 360                  * We are not going to put any of these on the list since it appears they have failed.
 361                  *
 362                  */
 363                 } else {
 364                     logger.debug("INFO: DesignatedWaiter.run: PDP= {} "
 365                                     + "designated= {}, current= {}, "
 366                                     + "designatedPdpHasFailed= {}, "
 367                                     + "standbyStatus= {}", pdp.getPdpId(),
 368                                     pdp.isDesignated(), false, designatedPdpHasFailed, standbyStatus);
 369                     pdpNotDesignatedNotCurrent(pdp, standbyStatus);
 370                 }
 371             }
 372         }
 373
 374         private void pdpDesignatedCurrent(DroolsPdp pdp, String standbyStatus, List<DroolsPdp> listOfDesignated) {
 375             //It is current, but it could have a standbystatus=coldstandby / hotstandby
 376             //If so, we need to stand it down and demote it
 377             if (!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)) {
 378                 if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 379                     logger.debug("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 380                                     + "butstandbystatus is not providingservice. "
 381                                     + " Executing stateManagement.demote()" + "\n\n", myPdp.getPdpId());
 382                     // So, we must demote it
 383                     try {
 384                         demoteMyPdp(pdp, standbyStatus);
 385                     } catch (Exception e) {
 386                         logger.error("DesignatedWaiter.run: myPdp: {} "
 387                                 + "Caught Exception attempting to demote myPdp,"
 388                                 + "message= {}", myPdp.getPdpId(), e);
 389                     }
 390                 } else {
 391                     // Don't demote a remote PDP that is current.  It should catch itself
 392                     logger.debug("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 393                                     + "but standbystatus is not providingservice. "
 394                                     + " Cannot execute stateManagement.demote() "
 395                                     + "since it it is not myPdp\n\n",
 396                                     myPdp.getPdpId());
 397                 }
 398
 399             } else {
 400                 // If we get here, it is ok to be on the list
 401                 logger.debug("DesignatedWaiter.run: PDP= {} is designated, "
 402                                 + "current and {} Noting PDP as "
 403                                 + "designated, standbyStatus= {}",
 404                                 pdp.getPdpId(), standbyStatus, standbyStatus);
 405                 listOfDesignated.add(pdp);
 406             }
 407         }
 408
 409         private void demoteMyPdp(DroolsPdp pdp, String standbyStatus) throws Exception {
 410             /*
 411              * Keep the order like this. StateManagement is last since it triggers
 412              * controller shutdown. This will change isDesignated and it can enter another
 413              * if-combination below
 414              */
 415             pdpsConnector.standDownPdp(pdp.getPdpId());
 416             myPdp.setDesignated(false);
 417             isDesignated = false;
 418             if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 419                     || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 420                 /*
 421                  * Only demote it if it appears it has not already been demoted. Don't worry
 422                  * about synching with the topic endpoint states.  That is done by the
 423                  * refreshStateAudit
 424                  */
 425                 stateManagementFeature.demote();
 426             }
 427         }
 428
 429         private void pdpDesignatedNotCurrent(DroolsPdp pdp) {
 430             /*
 431              * Changes designated to 0 but it is still potentially providing service.
 432              * Will affect isDesignated, so, it can enter an if-combination below
 433              */
 434             pdpsConnector.standDownPdp(pdp.getPdpId());
 435
 436             //need to change standbystatus to coldstandby
 437             if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 438                 logger.debug("\n\nDesignatedWaiter.run: myPdp {} is not Current. "
 439                                 + " Executing stateManagement.disableFailed()\n\n", myPdp.getPdpId());
 440                 // We found that myPdp is designated but not current
 441                 // So, we must cause it to disableFail
 442                 try {
 443                     myPdp.setDesignated(false);
 444                     pdpsConnector.setDesignated(myPdp, false);
 445                     isDesignated = false;
 446                     stateManagementFeature.disableFailed();
 447                 } catch (Exception e) {
 448                     logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception "
 449                             + "attempting to disableFail myPdp {}, message= {}",
 450                             myPdp.getPdpId(), myPdp.getPdpId(), e);
 451                 }
 452             } else { //it is a remote PDP that is failed
 453                 logger.debug("\n\nDesignatedWaiter.run: PDP {} is not Current. "
 454                                 + " Executing stateManagement.disableFailed(otherResourceName)\n\n",
 455                                 pdp.getPdpId());
 456                 // We found a PDP is designated but not current
 457                 // We already called standdown(pdp) which will change designated to false
 458                 // Now we need to disableFail it to get its states in synch.  The standbyStatus
 459                 // should equal coldstandby
 460                 try {
 461                     stateManagementFeature.disableFailed(pdp.getPdpId());
 462                 } catch (Exception e) {
 463                     logger.error("DesignatedWaiter.run: for PDP {}  Caught Exception attempting to "
 464                             + "disableFail({}), message= {}",
 465                             pdp.getPdpId(), pdp.getPdpId(), e);
 466                 }
 467
 468             }
 469         }
 470
 471         private void pdpNotDesignatedCurrent(DroolsPdp pdp, boolean designatedPdpHasFailed, String standbyStatus,
 472                         List<DroolsPdp> listOfDesignated) {
 473             if (!(StateManagement.HOT_STANDBY.equals(standbyStatus)
 474                     || StateManagement.COLD_STANDBY.equals(standbyStatus))) {
 475                 logger.debug("\n\nDesignatedWaiter.run: PDP {}"
 476                                 + " is NOT designated but IS current and"
 477                                 + " has a standbystatus= {}", pdp.getPdpId(), standbyStatus);
 478                 // Since it is current, we assume it can adjust its own state.
 479                 // We will demote if it is myPdp
 480                 if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 481                     //demote it
 482                     logger.debug("DesignatedWaiter.run: PDP {} going to "
 483                                     + "setDesignated = false and calling stateManagement.demote",
 484                                     pdp.getPdpId());
 485                     try {
 486                         //Keep the order like this.
 487                         //StateManagement is last since it triggers controller shutdown
 488                         pdpsConnector.setDesignated(myPdp, false);
 489                         myPdp.setDesignated(false);
 490                         isDesignated = false;
 491                         //This is definitely not a redundant call.
 492                         //It is attempting to correct a problem
 493                         stateManagementFeature.demote();
 494                         //recheck the standbystatus
 495                         standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 496                     } catch (Exception e) {
 497                         logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception "
 498                                 + "attempting to demote myPdp {}, message = {}",  myPdp.getPdpId(),
 499                                 myPdp.getPdpId(), e);
 500                     }
 501
 502                 }
 503             }
 504             if (StateManagement.HOT_STANDBY.equals(standbyStatus) && designatedPdpHasFailed) {
 505                 //add it to the list
 506                 logger.debug("INFO: DesignatedWaiter.run: PDP= {}"
 507                                 + " is not designated but is {} and designated PDP "
 508                                 + "has failed.  standbyStatus= {}", pdp.getPdpId(),
 509                                 standbyStatus, standbyStatus);
 510                 listOfDesignated.add(pdp);
 511             }
 512         }
 513
 514         private void pdpNotDesignatedNotCurrent(DroolsPdp pdp, String standbyStatus) {
 515             if (StateManagement.COLD_STANDBY.equals(standbyStatus)) {
 516                 return;
 517             }
 518
 519             //stand it down
 520             //disableFail it
 521             pdpsConnector.standDownPdp(pdp.getPdpId());
 522             if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 523                 /*
 524                  * I don't actually know how this condition could
 525                  * happen, but if it did, we would want to declare it
 526                  * failed.
 527                  */
 528                 logger.debug("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 529                                 + " Executing stateManagement.disableFailed()\n\n",
 530                                 myPdp.getPdpId());
 531                 // So, we must disableFail it
 532                 try {
 533                     //Keep the order like this.
 534                     //StateManagement is last since it triggers controller shutdown
 535                     pdpsConnector.setDesignated(myPdp, false);
 536                     myPdp.setDesignated(false);
 537                     isDesignated = false;
 538                     stateManagementFeature.disableFailed();
 539                 } catch (Exception e) {
 540                     logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 541                             + "disableFail myPdp {}, message= {}",
 542                             myPdp.getPdpId(), myPdp.getPdpId(), e);
 543                 }
 544             } else { //it is remote
 545                 logger.debug("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 546                                 + " Executing stateManagement.disableFailed({})\n\n",
 547                                 myPdp.getPdpId(), pdp.getPdpId());
 548                 // We already called standdown(pdp) which will change designated to false
 549                 // Now we need to disableFail it to get its states in sync.
 550                 // StandbyStatus = coldstandby
 551                 try {
 552                     stateManagementFeature.disableFailed(pdp.getPdpId());
 553                 } catch (Exception e) {
 554                     logger.error("DesignatedWaiter.run: for PDP {}"
 555                             + " Caught Exception attempting to disableFail({})"
 556                             + ", message=", pdp.getPdpId(), pdp.getPdpId(), e);
 557                 }
 558             }
 559         }
 560
 561         private void designateNoPdp() {
 562             // Just to be sure the parameters are correctly set
 563             myPdp.setDesignated(false);
 564             pdpsConnector.setDesignated(myPdp, false);
 565             isDesignated = false;
 566
 567             waitTimerLastRunDate = currentTime.getDate();
 568             logger.debug("DesignatedWaiter.run (designatedPdp == null) waitTimerLastRunDate = {}",
 569                             waitTimerLastRunDate);
 570             myPdp.setUpdatedDate(waitTimerLastRunDate);
 571             pdpsConnector.update(myPdp);
 572         }
 573
 574         private void designateMyPdp() {
 575             /*
 576              * update function expects myPdp.isDesignated to be true.
 577              */
 578             try {
 579                 //Keep the order like this.  StateManagement is last since it triggers controller init
 580                 myPdp.setDesignated(true);
 581                 myPdp.setDesignatedDate(currentTime.getDate());
 582                 pdpsConnector.setDesignated(myPdp, true);
 583                 isDesignated = true;
 584                 String standbyStatus = stateManagementFeature.getStandbyStatus();
 585                 if (!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)) {
 586                     /*
 587                      * Only call promote if it is not already in the right state.  Don't worry about
 588                      * synching the lower level topic endpoint states.  That is done by the
 589                      * refreshStateAudit.
 590                      * Note that we need to fetch the session list from 'mostRecentPrimary'
 591                      * at this point -- soon, 'mostRecentPrimary' will be set to this host.
 592                      */
 593                     //this.sessions = mostRecentPrimary.getSessions();
 594                     stateManagementFeature.promote();
 595                 }
 596             } catch (Exception e) {
 597                 logger.error("ERROR: DesignatedWaiter.run: Caught Exception attempting to promote PDP={}"
 598                         + ", message=", myPdp.getPdpId(), e);
 599                 myPdp.setDesignated(false);
 600                 pdpsConnector.setDesignated(myPdp, false);
 601                 isDesignated = false;
 602                 //If you can't promote it, demote it
 603                 try {
 604                     String standbyStatus = stateManagementFeature.getStandbyStatus();
 605                     if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 606                             || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 607                         /*
 608                          * Only call demote if it is not already in the right state.  Don't worry about
 609                          * synching the lower level topic endpoint states.  That is done by the
 610                          * refreshStateAudit.
 611                          */
 612                         stateManagementFeature.demote();
 613                     }
 614                 } catch (Exception e1) {
 615                     logger.error("ERROR: DesignatedWaiter.run: Caught StandbyStatusException "
 616                             + "attempting to promote then demote PDP={}, message=",
 617                             myPdp.getPdpId(), e1);
 618                 }
 619
 620             }
 621             waitTimerLastRunDate = currentTime.getDate();
 622             logger.debug("DesignatedWaiter.run (designatedPdp.getPdpId().equals(myPdp.getPdpId())) "
 623                             + "waitTimerLastRunDate = {}", waitTimerLastRunDate);
 624             myPdp.setUpdatedDate(waitTimerLastRunDate);
 625             pdpsConnector.update(myPdp);
 626         }
 627     }
 628
 629     /**
 630      * Sanitize designated list.
 631      *
 632      * @param listOfDesignated list of designated pdps
 633      * @return list of drools pdps
 634      */
 635     public List<DroolsPdp> santizeDesignatedList(List<DroolsPdp> listOfDesignated) {
 636
 637         boolean containsDesignated = false;
 638         boolean containsHotStandby = false;
 639         List<DroolsPdp> listForRemoval = new ArrayList<>();
 640         for (DroolsPdp pdp : listOfDesignated) {
 641             logger.debug("DesignatedWaiter.run sanitizing: pdp = {}"
 642                             + " isDesignated = {}", pdp.getPdpId(), pdp.isDesignated());
 643             if (pdp.isDesignated()) {
 644                 containsDesignated = true;
 645             } else {
 646                 containsHotStandby = true;
 647                 listForRemoval.add(pdp);
 648             }
 649         }
 650         if (containsDesignated && containsHotStandby) {
 651             //remove the hot standby from the list
 652             listOfDesignated.removeAll(listForRemoval);
 653         }
 654         return listOfDesignated;
 655     }
 656
 657     /**
 658      * Compute most recent primary.
 659      *
 660      * @param pdps collection of pdps
 661      * @param listOfDesignated list of designated pdps
 662      * @return drools pdp object
 663      */
 664     public DroolsPdp computeMostRecentPrimary(Collection<DroolsPdp> pdps, List<DroolsPdp> listOfDesignated) {
 665         boolean containsDesignated = listOfDesignated.stream().anyMatch(DroolsPdp::isDesignated);
 666
 667         DroolsPdp mostRecentPrimary = new DroolsPdpImpl(null, true, 1, new Date(0));
 668         mostRecentPrimary.setSite(null);
 669         logger.debug("DesignatedWaiter.run listOfDesignated.size() = {}", listOfDesignated.size());
 670
 671         if (listOfDesignated.size() <= 1) {
 672             logger.debug("DesignatedWainter.run: listOfDesignated.size <=1");
 673             //Only one or none is designated or hot standby.  Choose the latest designated date
 674             mostRecentPrimary = getLatestDesignated(pdps, mostRecentPrimary);
 675
 676         } else if (listOfDesignated.size() == pdps.size()) {
 677             logger.debug("DesignatedWainter.run: listOfDesignated.size = pdps.size() which is {}", pdps.size());
 678             //They are all designated or all hot standby.
 679             mostRecentPrimary = getBestDesignated(pdps, containsDesignated);
 680
 681         } else {
 682             logger.debug("DesignatedWainter.run: Some but not all are designated or hot standby. ");
 683             logger.debug("DesignatedWainter.run: containsDesignated = {}", containsDesignated);
 684             //Some but not all are designated or hot standby.
 685             if (containsDesignated) {
 686                 /*
 687                  * The list only contains designated.  This is a problem.  It is most likely a race
 688                  * condition that resulted in two thinking they should be designated. Choose the
 689                  * site with the latest designated date for the pdp not included on the designated list.
 690                  * This should be the site that had the last designation before this race condition
 691                  * occurred.
 692                  */
 693                 mostRecentPrimary = getLatestUndesignated(pdps, mostRecentPrimary, listOfDesignated);
 694
 695             } else {
 696                 //The list only contains hot standby. Choose the site of the latest designated date
 697                 mostRecentPrimary = getLatestDesignated(pdps, mostRecentPrimary);
 698             }
 699         }
 700         return mostRecentPrimary;
 701     }
 702
 703     private DroolsPdp getBestDesignated(Collection<DroolsPdp> pdps, boolean containsDesignated) {
 704         DroolsPdp mostRecentPrimary;
 705         mostRecentPrimary = null;
 706         for (DroolsPdp pdp : pdps) {
 707             if (mostRecentPrimary == null) {
 708                 mostRecentPrimary = pdp;
 709                 continue;
 710             }
 711             if (containsDesignated) { //Choose the site of the first designated date
 712                 if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) < 0) {
 713                     mostRecentPrimary = pdp;
 714                     logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 715                 }
 716             } else { //Choose the site with the latest designated date
 717                 if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 718                     mostRecentPrimary = pdp;
 719                     logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 720                 }
 721             }
 722         }
 723         return mostRecentPrimary;
 724     }
 725
 726     private DroolsPdp getLatestUndesignated(Collection<DroolsPdp> pdps, DroolsPdp mostRecentPrimary,
 727                     List<DroolsPdp> listOfDesignated) {
 728         for (DroolsPdp pdp : pdps) {
 729             if (listOfDesignated.contains(pdp)) {
 730                 continue; //Don't consider this entry
 731             }
 732             if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 733                 mostRecentPrimary = pdp;
 734                 logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 735             }
 736         }
 737         return mostRecentPrimary;
 738     }
 739
 740     private DroolsPdp getLatestDesignated(Collection<DroolsPdp> pdps, DroolsPdp mostRecentPrimary) {
 741         for (DroolsPdp pdp : pdps) {
 742             logger.debug("DesignatedWaiter.run pdp = {}"
 743                             + " pdp.getDesignatedDate() = {}",
 744                             pdp.getPdpId(), pdp.getDesignatedDate());
 745             if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 746                 mostRecentPrimary = pdp;
 747                 logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 748             }
 749         }
 750         return mostRecentPrimary;
 751     }
 752
 753     /**
 754      * Compue designated pdp.
 755      *
 756      * @param listOfDesignated list of designated pdps
 757      * @param mostRecentPrimary most recent primary pdpd
 758      * @return drools pdp object
 759      */
 760     public DroolsPdp computeDesignatedPdp(List<DroolsPdp> listOfDesignated, DroolsPdp mostRecentPrimary) {
 761         if (listOfDesignated.isEmpty()) {
 762             logger.debug("\nDesignatedWaiter.run: myPdp: {} listOfDesignated is: EMPTY.", myPdp.getPdpId());
 763             return null;
 764         }
 765
 766         if (listOfDesignated.size() == 1) {
 767             logger.debug("\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 768                             + "has ONE entry. PDP ID: {}", myPdp.getPdpId(), listOfDesignated.get(0).getPdpId());
 769             return listOfDesignated.get(0);
 770         }
 771
 772         logger.debug("DesignatedWaiter.run: myPdp: {} listOfDesignated.size(): {}", myPdp.getPdpId(),
 773                         listOfDesignated.size());
 774         DesignatedData data = new DesignatedData();
 775         for (DroolsPdp pdp : listOfDesignated) {
 776             DroolsPdp rejectedPdp;
 777
 778             // We need to determine if another PDP is the lowest priority
 779             if (nullSafeEquals(pdp.getSite(), mostRecentPrimary.getSite())) {
 780                 rejectedPdp = data.compareSameSite(pdp);
 781             } else {
 782                 rejectedPdp = data.compareDifferentSite(pdp);
 783             }
 784             // If the rejectedPdp is myPdp, we need to stand it down and demote it.  Each pdp is responsible
 785             // for demoting itself
 786             if (rejectedPdp != null && nullSafeEquals(rejectedPdp.getPdpId(), myPdp.getPdpId())) {
 787                 logger.debug("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated myPdp ID: {}"
 788                                 + " is NOT the lowest priority.  Executing stateManagement.demote()\n\n",
 789                                 myPdp.getPdpId(),
 790                                 myPdp.getPdpId());
 791                 // We found that myPdp is on the listOfDesignated and it is not the lowest priority
 792                 // So, we must demote it
 793                 demoteMyPdp();
 794             }
 795         }
 796
 797         DroolsPdp lowestPriorityPdp = data.getLowestPriority();
 798
 799         //now we have a valid value for lowestPriorityPdp
 800         logger.debug("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 801                         + "found the LOWEST priority pdp ID: {} "
 802                         + " It is now the designatedPpd from the perspective of myPdp ID: {} \n\n",
 803                         myPdp.getPdpId(), lowestPriorityPdp.getPdpId(), myPdp);
 804         return lowestPriorityPdp;
 805
 806     }
 807
 808     private class DesignatedData {
 809         private DroolsPdp lowestPrioritySameSite = null;
 810         private DroolsPdp lowestPriorityDifferentSite = null;
 811
 812         private DroolsPdp compareSameSite(DroolsPdp pdp) {
 813             if (lowestPrioritySameSite == null) {
 814                 if (lowestPriorityDifferentSite != null) {
 815                     //we need to reject lowestPriorityDifferentSite
 816                     DroolsPdp rejectedPdp = lowestPriorityDifferentSite;
 817                     lowestPriorityDifferentSite = pdp;
 818                     return rejectedPdp;
 819                 }
 820                 lowestPrioritySameSite = pdp;
 821                 return null;
 822             } else {
 823                 if (pdp.getPdpId().equals((lowestPrioritySameSite.getPdpId()))) {
 824                     return null;    //nothing to compare
 825                 }
 826                 if (pdp.comparePriority(lowestPrioritySameSite) < 0) {
 827                     logger.debug("\nDesignatedWaiter.run: myPdp {}  listOfDesignated pdp ID: {}"
 828                                     + " has lower priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 829                                     lowestPrioritySameSite.getPdpId());
 830                     //we need to reject lowestPrioritySameSite
 831                     DroolsPdp rejectedPdp = lowestPrioritySameSite;
 832                     lowestPrioritySameSite = pdp;
 833                     return rejectedPdp;
 834                 } else {
 835                     //we need to reject pdp and keep lowestPrioritySameSite
 836                     logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {} "
 837                                     + " has higher priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 838                                     lowestPrioritySameSite.getPdpId());
 839                     return pdp;
 840                 }
 841             }
 842         }
 843
 844         private DroolsPdp compareDifferentSite(DroolsPdp pdp) {
 845             if (lowestPrioritySameSite != null) {
 846                 //if we already have a candidate for same site, we don't want to bother with different sites
 847                 return pdp;
 848             } else {
 849                 if (lowestPriorityDifferentSite == null) {
 850                     lowestPriorityDifferentSite = pdp;
 851                     return null;
 852                 }
 853                 if (pdp.getPdpId().equals((lowestPriorityDifferentSite.getPdpId()))) {
 854                     return null;    //nothing to compare
 855                 }
 856                 if (pdp.comparePriority(lowestPriorityDifferentSite) < 0) {
 857                     logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 858                                     + " has lower priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 859                                     lowestPriorityDifferentSite.getPdpId());
 860                     //we need to reject lowestPriorityDifferentSite
 861                     DroolsPdp rejectedPdp = lowestPriorityDifferentSite;
 862                     lowestPriorityDifferentSite = pdp;
 863                     return rejectedPdp;
 864                 } else {
 865                     //we need to reject pdp and keep lowestPriorityDifferentSite
 866                     logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 867                                     + " has higher priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 868                                     lowestPriorityDifferentSite.getPdpId());
 869                     return pdp;
 870                 }
 871             }
 872         }
 873
 874         private DroolsPdp getLowestPriority() {
 875             return (lowestPrioritySameSite != null ? lowestPrioritySameSite : lowestPriorityDifferentSite);
 876         }
 877     }
 878
 879     private void demoteMyPdp() {
 880         try {
 881             //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 882             myPdp.setDesignated(false);
 883             pdpsConnector.setDesignated(myPdp, false);
 884             isDesignated = false;
 885             String standbyStatus = stateManagementFeature.getStandbyStatus();
 886             if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 887                     || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 888                 /*
 889                  * Only call demote if it is not already in the right state.  Don't worry about
 890                  * synching the lower level topic endpoint states.  That is done by the
 891                  * refreshStateAudit.
 892                  */
 893                 stateManagementFeature.demote();
 894             }
 895         } catch (Exception e) {
 896             myPdp.setDesignated(false);
 897             pdpsConnector.setDesignated(myPdp, false);
 898             isDesignated = false;
 899             logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 900                     + "demote myPdp {} myPdp.getPdpId(), message= {}", myPdp.getPdpId(),
 901                     e);
 902         }
 903     }
 904
 905     private class TimerUpdateClass extends TimerTask {
 906
 907         @Override
 908         public void run() {
 909             try {
 910                 logger.debug("TimerUpdateClass.run: entry");
 911                 checkWaitTimer();
 912             } catch (Exception e) {
 913                 logger.error("TimerUpdateClass.run caught an unexpected exception: ", e);
 914             }
 915             logger.debug("TimerUpdateClass.run.exit");
 916         }
 917     }
 918
 919     @Override
 920     public void checkThreadStatus() {
 921         checkWaitTimer();
 922     }
 923
 924     private void checkWaitTimer() {
 925         synchronized (checkWaitTimerLock) {
 926             try {
 927                 logger.debug("checkWaitTimer: entry");
 928                 Date now = currentTime.getDate();
 929                 long nowMs = now.getTime();
 930                 long waitTimerMs = waitTimerLastRunDate.getTime();
 931
 932                 //give it 10 times leeway
 933                 if ((nowMs - waitTimerMs)  > 10 * pdpUpdateInterval) {
 934                     if (allSeemsWell == null || allSeemsWell) {
 935                         allSeemsWell = false;
 936                         logger.debug("checkWaitTimer: calling allSeemsWell with ALLNOTWELL param");
 937                         stateManagementFeature.allSeemsWell(this.getClass().getName(),
 938                                 StateManagementFeatureApiConstants.ALLNOTWELL_STATE,
 939                                 "DesignationWaiter/ElectionHandler has STALLED");
 940                     }
 941                     logger.error("checkWaitTimer: nowMs - waitTimerMs = {}"
 942                             + ", exceeds 10* pdpUpdateInterval = {}"
 943                             + " DesignationWaiter is STALLED!", (nowMs - waitTimerMs), (10 * pdpUpdateInterval));
 944                 } else if (allSeemsWell == null || !allSeemsWell) {
 945                     allSeemsWell = true;
 946                     stateManagementFeature.allSeemsWell(this.getClass().getName(),
 947                             StateManagementFeatureApiConstants.ALLSEEMSWELL_STATE,
 948                             "DesignationWaiter/ElectionHandler has RESUMED");
 949                     logger.info("DesignationWaiter/ElectionHandler has RESUMED");
 950                 }
 951                 logger.debug("checkWaitTimer: exit");
 952             } catch (Exception e) {
 953                 logger.error("checkWaitTimer: caught unexpected exception: ", e);
 954             }
 955         }
 956     }
 957
 958     private long getDWaiterStartMs() {
 959         Date now = currentTime.getDate();
 960
 961         // Retrieve the ms since the epoch
 962         long nowMs = now.getTime();
 963
 964         // Time since the end of the last pdpUpdateInterval multiple
 965         long nowModMs = nowMs % pdpUpdateInterval;
 966
 967         // Time to the start of the next pdpUpdateInterval multiple
 968         long startMs = 2 * pdpUpdateInterval - nowModMs;
 969
 970         // Give the start time a minimum of a 5 second cushion
 971         if (startMs < 5000) {
 972             // Start at the beginning  of following interval
 973             startMs = pdpUpdateInterval + startMs;
 974         }
 975         return startMs;
 976     }
 977
 978     private boolean nullSafeEquals(Object one, Object two) {
 979         if (one == null && two == null) {
 980             return true;
 981         }
 982         if (one != null && two != null) {
 983             return one.equals(two);
 984         }
 985         return false;
 986     }
 987
 988     public String getPdpdNowActive() {
 989         return pdpdNowActive;
 990     }
 991
 992     public String getPdpdLastActive() {
 993         return pdpdLastActive;
 994     }
 995 }