gerrit.onap Code Review - policy/drools-pdp.git/blob

   1 /*
   2  * ============LICENSE_START=======================================================
   3  * feature-active-standby-management
   4  * ================================================================================
   5  * Copyright (C) 2017-2019 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.policy.drools.activestandby;
  22
  23 import java.util.ArrayList;
  24 import java.util.Collection;
  25 import java.util.Date;
  26 import java.util.List;
  27 import java.util.Timer;
  28 import java.util.TimerTask;
  29 import org.onap.policy.common.im.MonitorTime;
  30 import org.onap.policy.common.im.StateManagement;
  31 import org.onap.policy.common.utils.time.CurrentTime;
  32 import org.onap.policy.drools.statemanagement.StateManagementFeatureApi;
  33 import org.onap.policy.drools.statemanagement.StateManagementFeatureApiConstants;
  34 import org.slf4j.Logger;
  35 import org.slf4j.LoggerFactory;
  36
  37 public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
  38     private static final String RUN_PRIMARY_MSG = "DesignatedWaiter.run mostRecentPrimary = {}";
  39
  40     // get an instance of logger
  41     private static final Logger  logger = LoggerFactory.getLogger(DroolsPdpsElectionHandler.class);
  42     private DroolsPdpsConnector pdpsConnector;
  43     private Object checkWaitTimerLock = new Object();
  44     private Object designationWaiterLock = new Object();
  45
  46     /*
  47      * Must be static, so it can be referenced by JpaDroolsPdpsConnector,
  48      * without requiring a reference to the election handler instantiation.
  49      */
  50     private static DroolsPdp myPdp;
  51
  52     private Date waitTimerLastRunDate;
  53
  54     // The interval between runs of the DesignationWaiter
  55     private int pdpUpdateInterval;
  56
  57     private volatile boolean isDesignated;
  58
  59     private String pdpdNowActive;
  60     private String pdpdLastActive;
  61
  62     /*
  63      * Start allSeemsWell with a value of null so that, on the first run
  64      * of the checkWaitTimer it will set the value in IntegrityMonitor
  65      * regardless of whether it needs to be set to true or false.
  66      */
  67     private Boolean allSeemsWell = null;
  68
  69     private StateManagementFeatureApi stateManagementFeature;
  70
  71     private final CurrentTime currentTime = MonitorTime.getInstance();
  72
  73     private static boolean isUnitTesting = false;
  74     private static boolean isStalled = false;
  75
  76     /**
  77      * Constructor.
  78      *
  79      * @param pdps connectors
  80      * @param myPdp pdp
  81      */
  82     public DroolsPdpsElectionHandler(DroolsPdpsConnector pdps, DroolsPdp myPdp) {
  83         if (pdps == null) {
  84             logger.error("DroolsPdpsElectinHandler(): pdpsConnector==null");
  85             throw new IllegalArgumentException("DroolsPdpsElectinHandler(): pdpsConnector==null");
  86         }
  87         if (myPdp == null) {
  88             logger.error("DroolsPdpsElectinHandler(): droolsPdp==null");
  89             throw new IllegalArgumentException("DroolsPdpsElectinHandler(): DroolsPdp==null");
  90         }
  91
  92         pdpdNowActive = null;
  93         pdpdLastActive = null;
  94         this.pdpsConnector = pdps;
  95         DroolsPdpsElectionHandler.myPdp = myPdp;
  96         this.isDesignated = false;
  97
  98         // The interval between checks of the DesignationWaiter to be sure it is running.
  99         int pdpCheckInterval = 3000;
 100         try {
 101             pdpCheckInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(
 102                     ActiveStandbyProperties.PDP_CHECK_INVERVAL));
 103         } catch (Exception e) {
 104             logger.error("Could not get pdpCheckInterval property. Using default {}", pdpCheckInterval, e);
 105         }
 106         pdpUpdateInterval = 2000;
 107         try {
 108             pdpUpdateInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(
 109                     ActiveStandbyProperties.PDP_UPDATE_INTERVAL));
 110         } catch (Exception e) {
 111             logger.error("Could not get pdpUpdateInterval property. Using default {} ", pdpUpdateInterval, e);
 112         }
 113
 114         Date now = currentTime.getDate();
 115
 116         // Retrieve the ms since the epoch
 117         final long nowMs = now.getTime();
 118
 119         // Create the timer which will update the updateDate in DroolsPdpEntity table.
 120         // This is the heartbeat
 121         Timer updateWorker = Factory.getInstance().makeTimer();
 122
 123         // Schedule the TimerUpdateClass to run at 100 ms and run at pdpCheckInterval ms thereafter
 124         // NOTE: The first run of the TimerUpdateClass results in myPdp being added to the
 125         // drools droolsPdpEntity table.
 126         updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
 127
 128         // Create the timer which will run the election algorithm
 129         Timer waitTimer = Factory.getInstance().makeTimer();
 130
 131         // Schedule it to start in startMs ms
 132         // (so it will run after the updateWorker and run at pdpUpdateInterval ms thereafter
 133         long startMs = getDWaiterStartMs();
 134         DesignationWaiter designationWaiter = new DesignationWaiter();
 135         waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
 136         waitTimerLastRunDate = new Date(nowMs + startMs);
 137
 138         //Get the StateManagementFeature instance
 139
 140         for (StateManagementFeatureApi feature : StateManagementFeatureApiConstants.getImpl().getList()) {
 141             if (feature.getResourceName().equals(myPdp.getPdpId())) {
 142                 logger.debug("DroolsPdpsElectionHandler: Found StateManagementFeature"
 143                                 + " with resourceName: {}", myPdp.getPdpId());
 144                 stateManagementFeature = feature;
 145                 break;
 146             }
 147         }
 148         if (stateManagementFeature == null) {
 149             logger.error("DroolsPdpsElectionHandler failed to initialize.  "
 150                     + "Unable to get instance of StateManagementFeatureApi "
 151                     + "with resourceID: {}", myPdp.getPdpId());
 152         }
 153     }
 154
 155     public static void setIsUnitTesting(boolean val) {
 156         isUnitTesting = val;
 157     }
 158
 159     public static void setIsStalled(boolean val) {
 160         isStalled = val;
 161     }
 162
 163     /**
 164      * When the JpaDroolsPdpsConnector.standDown() method is invoked, it needs
 165      * access to myPdp, so it can keep its designation status in sync with the
 166      * DB.
 167      *
 168      * @param designated is designated value
 169      */
 170     public static void setMyPdpDesignated(boolean designated) {
 171         logger.debug("setMyPdpDesignated: designated= {}", designated);
 172         myPdp.setDesignated(designated);
 173     }
 174
 175     private class DesignationWaiter extends TimerTask {
 176         // get an instance of logger
 177         private final Logger  logger = LoggerFactory.getLogger(DesignationWaiter.class);
 178
 179         @Override
 180         public void run() {
 181             try {
 182                 logger.debug("DesignatedWaiter.run: Entering");
 183
 184                 //This is for testing the checkWaitTimer
 185                 if (isUnitTesting && isStalled) {
 186                     logger.debug("DesignatedWaiter.run: isUnitTesting = {} isStalled = {}",
 187                                     isUnitTesting, isStalled);
 188                     return;
 189                 }
 190
 191                 synchronized (designationWaiterLock) {
 192
 193                     logger.debug("DesignatedWaiter.run: Entering synchronized block");
 194
 195                     //It is possible that multiple PDPs are designated lead.  So, we will make a list of all designated
 196                     //PDPs and then decide which one really should be designated at the end.
 197                     List<DroolsPdp> listOfDesignated = new ArrayList<>();
 198
 199                     Collection<DroolsPdp> pdps = pdpsConnector.getDroolsPdps();
 200
 201                     logger.debug("DesignatedWaiter.run: pdps.size= {}", pdps.size());
 202
 203                     //This is only true if all designated PDPs have failed
 204                     allPdpsFailed(pdps, listOfDesignated);
 205
 206                     /*
 207                      * We have checked the four combinations of isDesignated and isCurrent.  Where appropriate,
 208                      * we added the PDPs to the potential list of designated pdps
 209                      *
 210                      * We need to give priority to pdps on the same site that is currently being used
 211                      * First, however, we must sanitize the list of designated to make sure their are
 212                      * only designated members or non-designated members.  There should not be both in
 213                      * the list. Because there are real time delays, it is possible that both types could
 214                      * be on the list.
 215                      */
 216
 217                     listOfDesignated = santizeDesignatedList(listOfDesignated);
 218
 219                     /*
 220                      * We need to figure out the last pdp that was the primary so we can get the last site
 221                      * name and the last session numbers.  We need to create a "dummy" droolspdp since
 222                      * it will be used in later comparisons and cannot be null.
 223                      */
 224
 225                     DroolsPdp mostRecentPrimary = computeMostRecentPrimary(pdps, listOfDesignated);
 226
 227                     if (mostRecentPrimary != null) {
 228                         pdpdLastActive = mostRecentPrimary.getPdpId();
 229                     }
 230
 231
 232                     /*
 233                      * It is possible to get here with more than one pdp designated and providing service. This normally
 234                      * occurs when there is a race condition with multiple nodes coming up at the same time. If that is
 235                      * the case we must determine which one is the one that should be designated and which one should
 236                      * be demoted.
 237                      *
 238                      * It is possible to have 0, 1, 2 or more but not all, or all designated.
 239                      *   If we have one designated and current, we chose it and are done
 240                      *   If we have 2 or more, but not all, we must determine which one is in the same site as
 241                      *   the previously designated pdp.
 242                      */
 243                     DroolsPdp designatedPdp = computeDesignatedPdp(listOfDesignated, mostRecentPrimary);
 244
 245                     if (designatedPdp == null) {
 246                         logger.warn("WARNING: DesignatedWaiter.run: No viable PDP found to be Designated. "
 247                             + "designatedPdp still null.");
 248                         designateNoPdp();
 249                         return;
 250                     }
 251
 252                     pdpdNowActive = designatedPdp.getPdpId();
 253
 254                     if (pdpdNowActive.equals(myPdp.getPdpId())) {
 255                         logger.debug("DesignatedWaiter.run: designatedPdp is PDP={}", myPdp.getPdpId());
 256                         designateMyPdp();
 257                         return;
 258                     }
 259
 260                     isDesignated = false;
 261
 262                 } // end synchronized
 263                 logger.debug("DesignatedWaiter.run: myPdp: {}; Returning, isDesignated= {}",
 264                                 isDesignated, myPdp.getPdpId());
 265
 266                 Date tmpDate = currentTime.getDate();
 267                 logger.debug("DesignatedWaiter.run (end of run) waitTimerLastRunDate = {}", tmpDate);
 268
 269                 waitTimerLastRunDate = tmpDate;
 270                 myPdp.setUpdatedDate(waitTimerLastRunDate);
 271                 pdpsConnector.update(myPdp);
 272
 273             } catch (Exception e) {
 274                 logger.error("DesignatedWaiter.run caught an unexpected exception: ", e);
 275             }
 276         } // end run
 277
 278         private void allPdpsFailed(Collection<DroolsPdp> pdps, List<DroolsPdp> listOfDesignated) {
 279             boolean designatedPdpHasFailed = pdpsConnector.hasDesignatedPdpFailed(pdps);
 280             logger.debug("DesignatedWaiter.run: designatedPdpHasFailed= {}", designatedPdpHasFailed);
 281             for (DroolsPdp pdp : pdps) {
 282                 logger.debug("DesignatedWaiter.run: evaluating pdp ID: {}", pdp.getPdpId());
 283
 284                 /*
 285                  * Note: side effect of isPdpCurrent is that any stale but
 286                  * designated PDPs will be marked as un-designated.
 287                  */
 288                 boolean isCurrent = pdpsConnector.isPdpCurrent(pdp);
 289
 290                 /*
 291                  * We can't use stateManagement.getStandbyStatus() here, because
 292                  * we need the standbyStatus, not for this PDP, but for the PDP
 293                  * being processed by this loop iteration.
 294                  */
 295                 String standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 296                 if (standbyStatus == null) {
 297                     // Treat this case as a cold standby -- if we
 298                     // abort here, no sessions will be created in a
 299                     // single-node test environment.
 300                     standbyStatus = StateManagement.COLD_STANDBY;
 301                 }
 302                 logger.debug("DesignatedWaiter.run: PDP= {},  isCurrent= {}", pdp.getPdpId(), isCurrent);
 303
 304                 adjustPdp(pdp, isCurrent, designatedPdpHasFailed, standbyStatus, listOfDesignated);
 305
 306
 307             } // end pdps loop
 308         }
 309
 310         private void adjustPdp(DroolsPdp pdp, boolean isCurrent, boolean designatedPdpHasFailed, String standbyStatus,
 311                         List<DroolsPdp> listOfDesignated) {
 312             /*
 313              * There are 4 combinations of isDesignated and isCurrent.  We will examine each one in-turn
 314              * and evaluate the each pdp in the list of pdps against each combination.
 315              */
 316             if (pdp.isDesignated()) {
 317                 /*
 318                  * This is the first combination of isDesignated and isCurrent
 319                  */
 320                 if (isCurrent) {
 321                     pdpDesignatedCurrent(pdp, standbyStatus, listOfDesignated);
 322
 323                 /*
 324                  * The second combination of isDesignated and isCurrent
 325                  *
 326                  * PDP is designated but not current; it has failed.
 327                  * So we stand it down (it doesn't matter what
 328                  * its standbyStatus is). None of these go on the list.
 329                  */
 330                 } else {
 331                     logger.debug("INFO: DesignatedWaiter.run: PDP= {} is currently "
 332                                     + "designated but is not current; "
 333                                     + "it has failed.  Standing down.  standbyStatus= {}",
 334                                     pdp.getPdpId(), standbyStatus);
 335                     pdpDesignatedNotCurrent(pdp);
 336                 }
 337
 338             } else {
 339                 // NOT designated
 340
 341
 342                 /*
 343                  * The third combination of isDesignated and isCurrent
 344                  * /*
 345                  * If a PDP is not currently designated but is providing service
 346                  * (erroneous, but recoverable) or hot standby
 347                  * we can add it to the list of possible designated if all the designated have failed
 348                  */
 349                 if (isCurrent) {
 350                     pdpNotDesignatedCurrent(pdp, designatedPdpHasFailed, standbyStatus,
 351                                     listOfDesignated);
 352
 353                 /*
 354                  * The fourth combination of isDesignated and isCurrent
 355                  *
 356                  * We are not going to put any of these on the list since it appears they have failed.
 357                  *
 358                  */
 359                 } else {
 360                     logger.debug("INFO: DesignatedWaiter.run: PDP= {} "
 361                                     + "designated= {}, current= {}, "
 362                                     + "designatedPdpHasFailed= {}, "
 363                                     + "standbyStatus= {}",pdp.getPdpId(),
 364                                     pdp.isDesignated(), false, designatedPdpHasFailed, standbyStatus);
 365                     pdpNotDesignatedNotCurrent(pdp, standbyStatus);
 366                 }
 367             }
 368         }
 369
 370         private void pdpDesignatedCurrent(DroolsPdp pdp, String standbyStatus, List<DroolsPdp> listOfDesignated) {
 371             //It is current, but it could have a standbystatus=coldstandby / hotstandby
 372             //If so, we need to stand it down and demote it
 373             if (!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)) {
 374                 if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 375                     logger.debug("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 376                                     + "butstandbystatus is not providingservice. "
 377                                     + " Executing stateManagement.demote()" + "\n\n", myPdp.getPdpId());
 378                     // So, we must demote it
 379                     try {
 380                         demoteMyPdp(pdp, standbyStatus);
 381                     } catch (Exception e) {
 382                         logger.error("DesignatedWaiter.run: myPdp: {} "
 383                                 + "Caught Exception attempting to demote myPdp,"
 384                                 + "message= {}", myPdp.getPdpId(), e);
 385                     }
 386                 } else {
 387                     // Don't demote a remote PDP that is current.  It should catch itself
 388                     logger.debug("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 389                                     + "but standbystatus is not providingservice. "
 390                                     + " Cannot execute stateManagement.demote() "
 391                                     + "since it it is not myPdp\n\n",
 392                                     myPdp.getPdpId());
 393                 }
 394
 395             } else {
 396                 // If we get here, it is ok to be on the list
 397                 logger.debug("DesignatedWaiter.run: PDP= {} is designated, "
 398                                 + "current and {} Noting PDP as "
 399                                 + "designated, standbyStatus= {}",
 400                                 pdp.getPdpId(), standbyStatus, standbyStatus);
 401                 listOfDesignated.add(pdp);
 402             }
 403         }
 404
 405         private void demoteMyPdp(DroolsPdp pdp, String standbyStatus) throws Exception {
 406             //Keep the order like this.  StateManagement is last since it
 407             //triggers controller shutdown
 408             //This will change isDesignated and it can enter another if(combination) below
 409             pdpsConnector.standDownPdp(pdp.getPdpId());
 410             myPdp.setDesignated(false);
 411             isDesignated = false;
 412             if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 413                     || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 414                 /*
 415                  * Only demote it if it appears it has not already been demoted. Don't worry
 416                  * about synching with the topic endpoint states.  That is done by the
 417                  * refreshStateAudit
 418                  */
 419                 stateManagementFeature.demote();
 420             }
 421         }
 422
 423         private void pdpDesignatedNotCurrent(DroolsPdp pdp) {
 424             /*
 425              * Changes designated to 0 but it is still potentially providing service
 426              * Will affect isDesignated, so, it can enter an if(combination) below
 427              */
 428             pdpsConnector.standDownPdp(pdp.getPdpId());
 429
 430             //need to change standbystatus to coldstandby
 431             if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 432                 logger.debug("\n\nDesignatedWaiter.run: myPdp {} is not Current. "
 433                                 + " Executing stateManagement.disableFailed()\n\n", myPdp.getPdpId());
 434                 // We found that myPdp is designated but not current
 435                 // So, we must cause it to disableFail
 436                 try {
 437                     myPdp.setDesignated(false);
 438                     pdpsConnector.setDesignated(myPdp, false);
 439                     isDesignated = false;
 440                     stateManagementFeature.disableFailed();
 441                 } catch (Exception e) {
 442                     logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception "
 443                             + "attempting to disableFail myPdp {}, message= {}",
 444                             myPdp.getPdpId(), myPdp.getPdpId(), e);
 445                 }
 446             } else { //it is a remote PDP that is failed
 447                 logger.debug("\n\nDesignatedWaiter.run: PDP {} is not Current. "
 448                                 + " Executing stateManagement.disableFailed(otherResourceName)\n\n",
 449                                 pdp.getPdpId() );
 450                 // We found a PDP is designated but not current
 451                 // We already called standdown(pdp) which will change designated to false
 452                 // Now we need to disableFail it to get its states in synch.  The standbyStatus
 453                 // should equal coldstandby
 454                 try {
 455                     stateManagementFeature.disableFailed(pdp.getPdpId());
 456                 } catch (Exception e) {
 457                     logger.error("DesignatedWaiter.run: for PDP {}  Caught Exception attempting to "
 458                             + "disableFail({}), message= {}",
 459                             pdp.getPdpId(), pdp.getPdpId(), e);
 460                 }
 461
 462             }
 463         }
 464
 465         private void pdpNotDesignatedCurrent(DroolsPdp pdp, boolean designatedPdpHasFailed, String standbyStatus,
 466                         List<DroolsPdp> listOfDesignated) {
 467             if (!(StateManagement.HOT_STANDBY.equals(standbyStatus)
 468                     || StateManagement.COLD_STANDBY.equals(standbyStatus))) {
 469                 logger.debug("\n\nDesignatedWaiter.run: PDP {}"
 470                                 + " is NOT designated but IS current and"
 471                                 + " has a standbystatus= {}", pdp.getPdpId(), standbyStatus);
 472                 // Since it is current, we assume it can adjust its own state.
 473                 // We will demote if it is myPdp
 474                 if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 475                     //demote it
 476                     logger.debug("DesignatedWaiter.run: PDP {} going to "
 477                                     + "setDesignated = false and calling stateManagement.demote",
 478                                     pdp.getPdpId());
 479                     try {
 480                         //Keep the order like this.
 481                         //StateManagement is last since it triggers controller shutdown
 482                         pdpsConnector.setDesignated(myPdp, false);
 483                         myPdp.setDesignated(false);
 484                         isDesignated = false;
 485                         //This is definitely not a redundant call.
 486                         //It is attempting to correct a problem
 487                         stateManagementFeature.demote();
 488                         //recheck the standbystatus
 489                         standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 490                     } catch (Exception e) {
 491                         logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception "
 492                                 + "attempting to demote myPdp {}, message = {}",  myPdp.getPdpId(),
 493                                 myPdp.getPdpId(), e);
 494                     }
 495
 496                 }
 497             }
 498             if (StateManagement.HOT_STANDBY.equals(standbyStatus) && designatedPdpHasFailed) {
 499                 //add it to the list
 500                 logger.debug("INFO: DesignatedWaiter.run: PDP= {}"
 501                                 + " is not designated but is {} and designated PDP "
 502                                 + "has failed.  standbyStatus= {}", pdp.getPdpId(),
 503                                 standbyStatus, standbyStatus);
 504                 listOfDesignated.add(pdp);
 505             }
 506         }
 507
 508         private void pdpNotDesignatedNotCurrent(DroolsPdp pdp, String standbyStatus) {
 509             if (StateManagement.COLD_STANDBY.equals(standbyStatus)) {
 510                 return;
 511             }
 512
 513             //stand it down
 514             //disableFail it
 515             pdpsConnector.standDownPdp(pdp.getPdpId());
 516             if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 517                 /*
 518                  * I don't actually know how this condition could
 519                  * happen, but if it did, we would want to declare it
 520                  * failed.
 521                  */
 522                 logger.debug("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 523                                 + " Executing stateManagement.disableFailed()\n\n",
 524                                 myPdp.getPdpId());
 525                 // So, we must disableFail it
 526                 try {
 527                     //Keep the order like this.
 528                     //StateManagement is last since it triggers controller shutdown
 529                     pdpsConnector.setDesignated(myPdp, false);
 530                     myPdp.setDesignated(false);
 531                     isDesignated = false;
 532                     stateManagementFeature.disableFailed();
 533                 } catch (Exception e) {
 534                     logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 535                             + "disableFail myPdp {}, message= {}",
 536                             myPdp.getPdpId(), myPdp.getPdpId(), e);
 537                 }
 538             } else { //it is remote
 539                 logger.debug("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 540                                 + " Executing stateManagement.disableFailed({})\n\n",
 541                                 myPdp.getPdpId(), pdp.getPdpId());
 542                 // We already called standdown(pdp) which will change designated to false
 543                 // Now we need to disableFail it to get its states in sync.
 544                 // StandbyStatus = coldstandby
 545                 try {
 546                     stateManagementFeature.disableFailed(pdp.getPdpId());
 547                 } catch (Exception e) {
 548                     logger.error("DesignatedWaiter.run: for PDP {}"
 549                             + " Caught Exception attempting to disableFail({})"
 550                             + ", message=", pdp.getPdpId(), pdp.getPdpId(), e);
 551                 }
 552             }
 553         }
 554
 555         private void designateNoPdp() {
 556             // Just to be sure the parameters are correctly set
 557             myPdp.setDesignated(false);
 558             pdpsConnector.setDesignated(myPdp,false);
 559             isDesignated = false;
 560
 561             waitTimerLastRunDate = currentTime.getDate();
 562             logger.debug("DesignatedWaiter.run (designatedPdp == null) waitTimerLastRunDate = {}",
 563                             waitTimerLastRunDate);
 564             myPdp.setUpdatedDate(waitTimerLastRunDate);
 565             pdpsConnector.update(myPdp);
 566         }
 567
 568         private void designateMyPdp() {
 569             /*
 570              * update function expects myPdp.isDesignated to be true.
 571              */
 572             try {
 573                 //Keep the order like this.  StateManagement is last since it triggers controller init
 574                 myPdp.setDesignated(true);
 575                 myPdp.setDesignatedDate(currentTime.getDate());
 576                 pdpsConnector.setDesignated(myPdp, true);
 577                 isDesignated = true;
 578                 String standbyStatus = stateManagementFeature.getStandbyStatus();
 579                 if (!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)) {
 580                     /*
 581                      * Only call promote if it is not already in the right state.  Don't worry about
 582                      * synching the lower level topic endpoint states.  That is done by the
 583                      * refreshStateAudit.
 584                      * Note that we need to fetch the session list from 'mostRecentPrimary'
 585                      * at this point -- soon, 'mostRecentPrimary' will be set to this host.
 586                      */
 587                     //this.sessions = mostRecentPrimary.getSessions();
 588                     stateManagementFeature.promote();
 589                 }
 590             } catch (Exception e) {
 591                 logger.error("ERROR: DesignatedWaiter.run: Caught Exception attempting to promote PDP={}"
 592                         + ", message=", myPdp.getPdpId(), e);
 593                 myPdp.setDesignated(false);
 594                 pdpsConnector.setDesignated(myPdp,false);
 595                 isDesignated = false;
 596                 //If you can't promote it, demote it
 597                 try {
 598                     String standbyStatus = stateManagementFeature.getStandbyStatus();
 599                     if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 600                             || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 601                         /*
 602                          * Only call demote if it is not already in the right state.  Don't worry about
 603                          * synching the lower level topic endpoint states.  That is done by the
 604                          * refreshStateAudit.
 605                          */
 606                         stateManagementFeature.demote();
 607                     }
 608                 } catch (Exception e1) {
 609                     logger.error("ERROR: DesignatedWaiter.run: Caught StandbyStatusException "
 610                             + "attempting to promote then demote PDP={}, message=",
 611                             myPdp.getPdpId(), e1);
 612                 }
 613
 614             }
 615             waitTimerLastRunDate = currentTime.getDate();
 616             logger.debug("DesignatedWaiter.run (designatedPdp.getPdpId().equals(myPdp.getPdpId())) "
 617                             + "waitTimerLastRunDate = " + waitTimerLastRunDate);
 618             myPdp.setUpdatedDate(waitTimerLastRunDate);
 619             pdpsConnector.update(myPdp);
 620         }
 621     }
 622
 623     /**
 624      * Sanitize designated list.
 625      *
 626      * @param listOfDesignated list of designated pdps
 627      * @return list of drools pdps
 628      */
 629     public List<DroolsPdp> santizeDesignatedList(List<DroolsPdp> listOfDesignated) {
 630
 631         boolean containsDesignated = false;
 632         boolean containsHotStandby = false;
 633         List<DroolsPdp> listForRemoval = new ArrayList<>();
 634         for (DroolsPdp pdp : listOfDesignated) {
 635             logger.debug("DesignatedWaiter.run sanitizing: pdp = {}"
 636                             + " isDesignated = {}",pdp.getPdpId(), pdp.isDesignated());
 637             if (pdp.isDesignated()) {
 638                 containsDesignated = true;
 639             } else {
 640                 containsHotStandby = true;
 641                 listForRemoval.add(pdp);
 642             }
 643         }
 644         if (containsDesignated && containsHotStandby) {
 645             //remove the hot standby from the list
 646             listOfDesignated.removeAll(listForRemoval);
 647         }
 648         return listOfDesignated;
 649     }
 650
 651     /**
 652      * Compute most recent primary.
 653      *
 654      * @param pdps collection of pdps
 655      * @param listOfDesignated list of designated pdps
 656      * @return drools pdp object
 657      */
 658     public DroolsPdp computeMostRecentPrimary(Collection<DroolsPdp> pdps, List<DroolsPdp> listOfDesignated) {
 659         boolean containsDesignated = listOfDesignated.stream().anyMatch(DroolsPdp::isDesignated);
 660
 661         DroolsPdp mostRecentPrimary = new DroolsPdpImpl(null, true, 1, new Date(0));
 662         mostRecentPrimary.setSite(null);
 663         logger.debug("DesignatedWaiter.run listOfDesignated.size() = {}", listOfDesignated.size());
 664
 665         if (listOfDesignated.size() <= 1) {
 666             logger.debug("DesignatedWainter.run: listOfDesignated.size <=1");
 667             //Only one or none is designated or hot standby.  Choose the latest designated date
 668             mostRecentPrimary = getLatestDesignated(pdps, mostRecentPrimary);
 669
 670         } else if (listOfDesignated.size() == pdps.size()) {
 671             logger.debug("DesignatedWainter.run: listOfDesignated.size = pdps.size() which is {}", pdps.size());
 672             //They are all designated or all hot standby.
 673             mostRecentPrimary = getBestDesignated(pdps, containsDesignated);
 674
 675         } else {
 676             logger.debug("DesignatedWainter.run: Some but not all are designated or hot standby. ");
 677             logger.debug("DesignatedWainter.run: containsDesignated = {}", containsDesignated);
 678             //Some but not all are designated or hot standby.
 679             if (containsDesignated) {
 680                 /*
 681                  * The list only contains designated.  This is a problem.  It is most likely a race
 682                  * condition that resulted in two thinking they should be designated. Choose the
 683                  * site with the latest designated date for the pdp not included on the designated list.
 684                  * This should be the site that had the last designation before this race condition
 685                  * occurred.
 686                  */
 687                 mostRecentPrimary = getLatestUndesignated(pdps, mostRecentPrimary, listOfDesignated);
 688
 689             } else {
 690                 //The list only contains hot standby. Choose the site of the latest designated date
 691                 mostRecentPrimary = getLatestDesignated(pdps, mostRecentPrimary);
 692             }
 693         }
 694         return mostRecentPrimary;
 695     }
 696
 697     private DroolsPdp getBestDesignated(Collection<DroolsPdp> pdps, boolean containsDesignated) {
 698         DroolsPdp mostRecentPrimary;
 699         mostRecentPrimary = null;
 700         for (DroolsPdp pdp : pdps) {
 701             if (mostRecentPrimary == null) {
 702                 mostRecentPrimary = pdp;
 703                 continue;
 704             }
 705             if (containsDesignated) { //Choose the site of the first designated date
 706                 if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) < 0) {
 707                     mostRecentPrimary = pdp;
 708                     logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 709                 }
 710             } else { //Choose the site with the latest designated date
 711                 if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 712                     mostRecentPrimary = pdp;
 713                     logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 714                 }
 715             }
 716         }
 717         return mostRecentPrimary;
 718     }
 719
 720     private DroolsPdp getLatestUndesignated(Collection<DroolsPdp> pdps, DroolsPdp mostRecentPrimary,
 721                     List<DroolsPdp> listOfDesignated) {
 722         for (DroolsPdp pdp : pdps) {
 723             if (listOfDesignated.contains(pdp)) {
 724                 continue; //Don't consider this entry
 725             }
 726             if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 727                 mostRecentPrimary = pdp;
 728                 logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 729             }
 730         }
 731         return mostRecentPrimary;
 732     }
 733
 734     private DroolsPdp getLatestDesignated(Collection<DroolsPdp> pdps, DroolsPdp mostRecentPrimary) {
 735         for (DroolsPdp pdp : pdps) {
 736             logger.debug("DesignatedWaiter.run pdp = {}"
 737                             + " pdp.getDesignatedDate() = {}",
 738                             pdp.getPdpId(), pdp.getDesignatedDate());
 739             if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 740                 mostRecentPrimary = pdp;
 741                 logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 742             }
 743         }
 744         return mostRecentPrimary;
 745     }
 746
 747     /**
 748      * Compue designated pdp.
 749      *
 750      * @param listOfDesignated list of designated pdps
 751      * @param mostRecentPrimary most recent primary pdpd
 752      * @return drools pdp object
 753      */
 754     public DroolsPdp computeDesignatedPdp(List<DroolsPdp> listOfDesignated, DroolsPdp mostRecentPrimary) {
 755         if (listOfDesignated.isEmpty()) {
 756             logger.debug("\nDesignatedWaiter.run: myPdp: {} listOfDesignated is: EMPTY.", myPdp.getPdpId());
 757             return null;
 758         }
 759
 760         if (listOfDesignated.size() == 1) {
 761             logger.debug("\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 762                             + "has ONE entry. PDP ID: {}", myPdp.getPdpId(), listOfDesignated.get(0).getPdpId());
 763             return listOfDesignated.get(0);
 764         }
 765
 766         logger.debug("DesignatedWaiter.run: myPdp: {} listOfDesignated.size(): {}", myPdp.getPdpId(),
 767                         listOfDesignated.size());
 768         DesignatedData data = new DesignatedData();
 769         for (DroolsPdp pdp : listOfDesignated) {
 770             DroolsPdp rejectedPdp;
 771
 772             // We need to determine if another PDP is the lowest priority
 773             if (nullSafeEquals(pdp.getSite(), mostRecentPrimary.getSite())) {
 774                 rejectedPdp = data.compareSameSite(pdp);
 775             } else {
 776                 rejectedPdp = data.compareDifferentSite(pdp);
 777             }
 778             // If the rejectedPdp is myPdp, we need to stand it down and demote it.  Each pdp is responsible
 779             // for demoting itself
 780             if (rejectedPdp != null && nullSafeEquals(rejectedPdp.getPdpId(),myPdp.getPdpId())) {
 781                 logger.debug("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated myPdp ID: {}"
 782                                 + " is NOT the lowest priority.  Executing stateManagement.demote()\n\n",
 783                                 myPdp.getPdpId(),
 784                                 myPdp.getPdpId());
 785                 // We found that myPdp is on the listOfDesignated and it is not the lowest priority
 786                 // So, we must demote it
 787                 demoteMyPdp();
 788             }
 789         } //end: for(DroolsPdp pdp : listOfDesignated)
 790
 791         DroolsPdp lowestPriorityPdp = data.getLowestPriority();
 792
 793         //now we have a valid value for lowestPriorityPdp
 794         logger.debug("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 795                         + "found the LOWEST priority pdp ID: {} "
 796                         + " It is now the designatedPpd from the perspective of myPdp ID: {} \n\n",
 797                         myPdp.getPdpId(), lowestPriorityPdp.getPdpId(), myPdp);
 798         return lowestPriorityPdp;
 799
 800     }
 801
 802     private class DesignatedData {
 803         private DroolsPdp lowestPrioritySameSite = null;
 804         private DroolsPdp lowestPriorityDifferentSite = null;
 805
 806         private DroolsPdp compareSameSite(DroolsPdp pdp) {
 807             if (lowestPrioritySameSite == null) {
 808                 if (lowestPriorityDifferentSite != null) {
 809                     //we need to reject lowestPriorityDifferentSite
 810                     DroolsPdp rejectedPdp = lowestPriorityDifferentSite;
 811                     lowestPriorityDifferentSite = pdp;
 812                     return rejectedPdp;
 813                 }
 814                 lowestPrioritySameSite = pdp;
 815                 return null;
 816             } else {
 817                 if (pdp.getPdpId().equals((lowestPrioritySameSite.getPdpId()))) {
 818                     return null;//nothing to compare
 819                 }
 820                 if (pdp.comparePriority(lowestPrioritySameSite) < 0) {
 821                     logger.debug("\nDesignatedWaiter.run: myPdp {}  listOfDesignated pdp ID: {}"
 822                                     + " has lower priority than pdp ID: {}",myPdp.getPdpId(), pdp.getPdpId(),
 823                                     lowestPrioritySameSite.getPdpId());
 824                     //we need to reject lowestPrioritySameSite
 825                     DroolsPdp rejectedPdp = lowestPrioritySameSite;
 826                     lowestPrioritySameSite = pdp;
 827                     return rejectedPdp;
 828                 } else {
 829                     //we need to reject pdp and keep lowestPrioritySameSite
 830                     logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {} "
 831                                     + " has higher priority than pdp ID: {}", myPdp.getPdpId(),pdp.getPdpId(),
 832                                     lowestPrioritySameSite.getPdpId());
 833                     return pdp;
 834                 }
 835             }
 836         }
 837
 838         private DroolsPdp compareDifferentSite(DroolsPdp pdp) {
 839             if (lowestPrioritySameSite != null) {
 840                 //if we already have a candidate for same site, we don't want to bother with different sites
 841                 return pdp;
 842             } else {
 843                 if (lowestPriorityDifferentSite == null) {
 844                     lowestPriorityDifferentSite = pdp;
 845                     return null;
 846                 }
 847                 if (pdp.getPdpId().equals((lowestPriorityDifferentSite.getPdpId()))) {
 848                     return null;//nothing to compare
 849                 }
 850                 if (pdp.comparePriority(lowestPriorityDifferentSite) < 0) {
 851                     logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 852                                     + " has lower priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 853                                     lowestPriorityDifferentSite.getPdpId());
 854                     //we need to reject lowestPriorityDifferentSite
 855                     DroolsPdp rejectedPdp = lowestPriorityDifferentSite;
 856                     lowestPriorityDifferentSite = pdp;
 857                     return rejectedPdp;
 858                 } else {
 859                     //we need to reject pdp and keep lowestPriorityDifferentSite
 860                     logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 861                                     + " has higher priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 862                                     lowestPriorityDifferentSite.getPdpId());
 863                     return pdp;
 864                 }
 865             }
 866         }
 867
 868         private DroolsPdp getLowestPriority() {
 869             return (lowestPrioritySameSite != null ? lowestPrioritySameSite : lowestPriorityDifferentSite);
 870         }
 871     }
 872
 873     private void demoteMyPdp() {
 874         try {
 875             //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 876             myPdp.setDesignated(false);
 877             pdpsConnector.setDesignated(myPdp, false);
 878             isDesignated = false;
 879             String standbyStatus = stateManagementFeature.getStandbyStatus();
 880             if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 881                     || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 882                 /*
 883                  * Only call demote if it is not already in the right state.  Don't worry about
 884                  * synching the lower level topic endpoint states.  That is done by the
 885                  * refreshStateAudit.
 886                  */
 887                 stateManagementFeature.demote();
 888             }
 889         } catch (Exception e) {
 890             myPdp.setDesignated(false);
 891             pdpsConnector.setDesignated(myPdp, false);
 892             isDesignated = false;
 893             logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 894                     + "demote myPdp {} myPdp.getPdpId(), message= {}", myPdp.getPdpId(),
 895                     e);
 896         }
 897     }
 898
 899     private class TimerUpdateClass extends TimerTask {
 900
 901         @Override
 902         public void run() {
 903             try {
 904                 logger.debug("TimerUpdateClass.run: entry");
 905                 checkWaitTimer();
 906             } catch (Exception e) {
 907                 logger.error("TimerUpdateClass.run caught an unexpected exception: ", e);
 908             }
 909             logger.debug("TimerUpdateClass.run.exit");
 910         }
 911     }
 912
 913     @Override
 914     public void checkThreadStatus() {
 915         checkWaitTimer();
 916     }
 917
 918     private void checkWaitTimer() {
 919         synchronized (checkWaitTimerLock) {
 920             try {
 921                 logger.debug("checkWaitTimer: entry");
 922                 Date now = currentTime.getDate();
 923                 long nowMs = now.getTime();
 924                 long waitTimerMs = waitTimerLastRunDate.getTime();
 925
 926                 //give it 10 times leeway
 927                 if ((nowMs - waitTimerMs)  > 10 * pdpUpdateInterval) {
 928                     if (allSeemsWell == null || allSeemsWell) {
 929                         allSeemsWell = false;
 930                         logger.debug("checkWaitTimer: calling allSeemsWell with ALLNOTWELL param");
 931                         stateManagementFeature.allSeemsWell(this.getClass().getName(),
 932                                 StateManagementFeatureApiConstants.ALLNOTWELL_STATE,
 933                                 "DesignationWaiter/ElectionHandler has STALLED");
 934                     }
 935                     logger.error("checkWaitTimer: nowMs - waitTimerMs = {}"
 936                             + ", exceeds 10* pdpUpdateInterval = {}"
 937                             + " DesignationWaiter is STALLED!", (nowMs - waitTimerMs), (10 * pdpUpdateInterval));
 938                 } else if (allSeemsWell == null || !allSeemsWell) {
 939                     allSeemsWell = true;
 940                     stateManagementFeature.allSeemsWell(this.getClass().getName(),
 941                             StateManagementFeatureApiConstants.ALLSEEMSWELL_STATE,
 942                             "DesignationWaiter/ElectionHandler has RESUMED");
 943                     logger.info("DesignationWaiter/ElectionHandler has RESUMED");
 944                 }
 945                 logger.debug("checkWaitTimer: exit");
 946             } catch (Exception e) {
 947                 logger.error("checkWaitTimer: caught unexpected exception: ", e);
 948             }
 949         }
 950     }
 951
 952     private long getDWaiterStartMs() {
 953         Date now = currentTime.getDate();
 954
 955         // Retrieve the ms since the epoch
 956         long nowMs = now.getTime();
 957
 958         // Time since the end of the last pdpUpdateInterval multiple
 959         long nowModMs = nowMs % pdpUpdateInterval;
 960
 961         // Time to the start of the next pdpUpdateInterval multiple
 962         long startMs = 2 * pdpUpdateInterval - nowModMs;
 963
 964         // Give the start time a minimum of a 5 second cushion
 965         if (startMs < 5000) {
 966             // Start at the beginning  of following interval
 967             startMs = pdpUpdateInterval + startMs;
 968         }
 969         return startMs;
 970     }
 971
 972     private boolean nullSafeEquals(Object one, Object two) {
 973         if (one == null && two == null) {
 974             return true;
 975         }
 976         if (one != null && two != null) {
 977             return one.equals(two);
 978         }
 979         return false;
 980     }
 981
 982     public String getPdpdNowActive() {
 983         return pdpdNowActive;
 984     }
 985
 986     public String getPdpdLastActive() {
 987         return pdpdLastActive;
 988     }
 989 }