feature-active-standby-management/src/main/java/org/onap/policy/drools/activestandby/DroolsPdpsElectionHandler.java

   1 /*
   2  * ============LICENSE_START=======================================================
   3  * feature-active-standby-management
   4  * ================================================================================
   5  * Copyright (C) 2017-2019 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.policy.drools.activestandby;
  22
  23 import java.util.ArrayList;
  24 import java.util.Collection;
  25 import java.util.Date;
  26 import java.util.List;
  27 import java.util.Timer;
  28 import java.util.TimerTask;
  29
  30 import org.onap.policy.common.im.StateManagement;
  31 import org.onap.policy.drools.statemanagement.StateManagementFeatureApi;
  32 import org.slf4j.Logger;
  33 import org.slf4j.LoggerFactory;
  34
  35 public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
  36     // get an instance of logger
  37     private static final Logger  logger = LoggerFactory.getLogger(DroolsPdpsElectionHandler.class);
  38     private DroolsPdpsConnector pdpsConnector;
  39     private Object checkWaitTimerLock = new Object();
  40     private Object designationWaiterLock = new Object();
  41
  42     /*
  43      * Must be static, so it can be referenced by JpaDroolsPdpsConnector,
  44      * without requiring a reference to the election handler instantiation.
  45      */
  46     private static DroolsPdp myPdp;
  47
  48     private DesignationWaiter designationWaiter;
  49     private Timer updateWorker;
  50     private Timer waitTimer;
  51     private Date waitTimerLastRunDate;
  52
  53     // The interval between checks of the DesignationWaiter to be sure it is running.
  54     private int pdpCheckInterval;
  55
  56     // The interval between runs of the DesignationWaiter
  57     private int pdpUpdateInterval;
  58
  59     private volatile boolean isDesignated;
  60
  61     private String pdpdNowActive;
  62     private String pdpdLastActive;
  63
  64     /*
  65      * Start allSeemsWell with a value of null so that, on the first run
  66      * of the checkWaitTimer it will set the value in IntegrityMonitor
  67      * regardless of whether it needs to be set to true or false.
  68      */
  69     private Boolean allSeemsWell = null;
  70
  71     private StateManagementFeatureApi stateManagementFeature;
  72
  73     private static boolean isUnitTesting = false;
  74     private static boolean isStalled = false;
  75
  76     /**
  77      * Constructor.
  78      *
  79      * @param pdps connectors
  80      * @param myPdp pdp
  81      */
  82     public DroolsPdpsElectionHandler(DroolsPdpsConnector pdps, DroolsPdp myPdp) {
  83         if (pdps == null) {
  84             logger.error("DroolsPdpsElectinHandler(): pdpsConnector==null");
  85             throw new IllegalArgumentException("DroolsPdpsElectinHandler(): pdpsConnector==null");
  86         }
  87         if (myPdp == null) {
  88             logger.error("DroolsPdpsElectinHandler(): droolsPdp==null");
  89             throw new IllegalArgumentException("DroolsPdpsElectinHandler(): DroolsPdp==null");
  90         }
  91
  92         pdpdNowActive = null;
  93         pdpdLastActive = null;
  94         this.pdpsConnector = pdps;
  95         DroolsPdpsElectionHandler.myPdp = myPdp;
  96         this.isDesignated = false;
  97         pdpCheckInterval = 3000;
  98         try {
  99             pdpCheckInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(
 100                     ActiveStandbyProperties.PDP_CHECK_INVERVAL));
 101         } catch (Exception e) {
 102             logger.error("Could not get pdpCheckInterval property. Using default {}",pdpCheckInterval, e);
 103         }
 104         pdpUpdateInterval = 2000;
 105         try {
 106             pdpUpdateInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(
 107                     ActiveStandbyProperties.PDP_UPDATE_INTERVAL));
 108         } catch (Exception e) {
 109             logger.error("Could not get pdpUpdateInterval property. Using default {} ", pdpUpdateInterval, e);
 110         }
 111
 112         Date now = new Date();
 113
 114         // Retrieve the ms since the epoch
 115         final long nowMs = now.getTime();
 116
 117         // Create the timer which will update the updateDate in DroolsPdpEntity table.
 118         // This is the heartbeat
 119         updateWorker = new Timer();
 120
 121         // Schedule the TimerUpdateClass to run at 100 ms and run at pdpCheckInterval ms thereafter
 122         // NOTE: The first run of the TimerUpdateClass results in myPdp being added to the
 123         // drools droolsPdpEntity table.
 124         updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
 125
 126         // Create the timer which will run the election algorithm
 127         waitTimer = new Timer();
 128
 129         // Schedule it to start in startMs ms
 130         // (so it will run after the updateWorker and run at pdpUpdateInterval ms thereafter
 131         long startMs = getDWaiterStartMs();
 132         designationWaiter = new DesignationWaiter();
 133         waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
 134         waitTimerLastRunDate = new Date(nowMs + startMs);
 135
 136         //Get the StateManagementFeature instance
 137
 138         for (StateManagementFeatureApi feature : StateManagementFeatureApi.impl.getList()) {
 139             if (feature.getResourceName().equals(myPdp.getPdpId())) {
 140                 logger.debug("DroolsPdpsElectionHandler: Found StateManagementFeature"
 141                                 + " with resourceName: {}", myPdp.getPdpId());
 142                 stateManagementFeature = feature;
 143                 break;
 144             }
 145         }
 146         if (stateManagementFeature == null) {
 147             logger.error("DroolsPdpsElectionHandler failed to initialize.  "
 148                     + "Unable to get instance of StateManagementFeatureApi "
 149                     + "with resourceID: {}", myPdp.getPdpId());
 150         }
 151     }
 152
 153     public static void setIsUnitTesting(boolean val) {
 154         isUnitTesting = val;
 155     }
 156
 157     public static void setIsStalled(boolean val) {
 158         isStalled = val;
 159     }
 160
 161     /**
 162      * When the JpaDroolsPdpsConnector.standDown() method is invoked, it needs
 163      * access to myPdp, so it can keep its designation status in sync with the
 164      * DB.
 165      *
 166      * @param designated is designated value
 167      */
 168     public static void setMyPdpDesignated(boolean designated) {
 169         logger.debug("setMyPdpDesignated: designated= {}", designated);
 170         myPdp.setDesignated(designated);
 171     }
 172
 173     private class DesignationWaiter extends TimerTask {
 174         // get an instance of logger
 175         private final Logger  logger = LoggerFactory.getLogger(DesignationWaiter.class);
 176
 177         @Override
 178         public void run() {
 179             try {
 180                 logger.debug("DesignatedWaiter.run: Entering");
 181
 182                 //This is for testing the checkWaitTimer
 183                 if (isUnitTesting && isStalled) {
 184                     logger.debug("DesignatedWaiter.run: isUnitTesting = {} isStalled = {}",
 185                                     isUnitTesting, isStalled);
 186                     return;
 187                 }
 188
 189                 synchronized (designationWaiterLock) {
 190
 191                     logger.debug("DesignatedWaiter.run: Entering synchronized block");
 192
 193                     //It is possible that multiple PDPs are designated lead.  So, we will make a list of all designated
 194                     //PDPs and then decide which one really should be designated at the end.
 195                     List<DroolsPdp> listOfDesignated = new ArrayList<>();
 196
 197                     Collection<DroolsPdp> pdps = pdpsConnector.getDroolsPdps();
 198
 199                     logger.debug("DesignatedWaiter.run: pdps.size= {}", pdps.size());
 200
 201                     //This is only true if all designated PDPs have failed
 202                     boolean designatedPdpHasFailed = pdpsConnector.hasDesignatedPdpFailed(pdps);
 203                     logger.debug("DesignatedWaiter.run: designatedPdpHasFailed= {}", designatedPdpHasFailed);
 204                     for (DroolsPdp pdp : pdps) {
 205                         logger.debug("DesignatedWaiter.run: evaluating pdp ID: {}", pdp.getPdpId());
 206
 207                         /*
 208                          * Note: side effect of isPdpCurrent is that any stale but
 209                          * designated PDPs will be marked as un-designated.
 210                          */
 211                         boolean isCurrent = pdpsConnector.isPdpCurrent(pdp);
 212
 213                         /*
 214                          * We can't use stateManagement.getStandbyStatus() here, because
 215                          * we need the standbyStatus, not for this PDP, but for the PDP
 216                          * being processed by this loop iteration.
 217                          */
 218                         String standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 219                         if (standbyStatus == null) {
 220                             // Treat this case as a cold standby -- if we
 221                             // abort here, no sessions will be created in a
 222                             // single-node test environment.
 223                             standbyStatus = StateManagement.COLD_STANDBY;
 224                         }
 225                         logger.debug("DesignatedWaiter.run: PDP= {},  isCurrent= {}", pdp.getPdpId(), isCurrent);
 226
 227                         /*
 228                          * There are 4 combinations of isDesignated and isCurrent.  We will examine each one in-turn
 229                          * and evaluate the each pdp in the list of pdps against each combination.
 230                          *
 231                          * This is the first combination of isDesignated and isCurrent
 232                          */
 233                         if (pdp.isDesignated()  &&  isCurrent) {
 234                             //It is current, but it could have a standbystatus=coldstandby / hotstandby
 235                             //If so, we need to stand it down and demote it
 236                             if (!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)) {
 237                                 if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 238                                     logger.debug("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 239                                                     + "butstandbystatus is not providingservice. "
 240                                                     + " Executing stateManagement.demote()" + "\n\n", myPdp.getPdpId());
 241                                     // So, we must demote it
 242                                     try {
 243                                         //Keep the order like this.  StateManagement is last since it
 244                                         //triggers controller shutdown
 245                                         //This will change isDesignated and it can enter another if(combination) below
 246                                         pdpsConnector.standDownPdp(pdp.getPdpId());
 247                                         myPdp.setDesignated(false);
 248                                         isDesignated = false;
 249                                         if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 250                                                 || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 251                                             /*
 252                                              * Only demote it if it appears it has not already been demoted. Don't worry
 253                                              * about synching with the topic endpoint states.  That is done by the
 254                                              * refreshStateAudit
 255                                              */
 256                                             stateManagementFeature.demote();
 257                                         }
 258                                         //update the standbystatus to check in a later
 259                                         //combination of isDesignated and isCurrent
 260                                         standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 261                                     } catch (Exception e) {
 262                                         logger.error("DesignatedWaiter.run: myPdp: {} "
 263                                                 + "Caught Exception attempting to demote myPdp,"
 264                                                 + "message= {}", myPdp.getPdpId(), e);
 265                                     }
 266                                 } else {
 267                                     // Don't demote a remote PDP that is current.  It should catch itself
 268                                     logger.debug("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 269                                                     + "but standbystatus is not providingservice. "
 270                                                     + " Cannot execute stateManagement.demote() "
 271                                                     + "since it it is not myPdp\n\n",
 272                                                     myPdp.getPdpId());
 273                                 }
 274
 275                             } else {
 276                                 // If we get here, it is ok to be on the list
 277                                 logger.debug("DesignatedWaiter.run: PDP= {} is designated, "
 278                                                 + "current and {} Noting PDP as "
 279                                                 + "designated, standbyStatus= {}",
 280                                                 pdp.getPdpId(), standbyStatus, standbyStatus);
 281                                 listOfDesignated.add(pdp);
 282                             }
 283
 284
 285                         }
 286
 287
 288                         /*
 289                          * The second combination of isDesignated and isCurrent
 290                          *
 291                          * PDP is designated but not current; it has failed.
 292                          * So we stand it down (it doesn't matter what
 293                          * its standbyStatus is). None of these go on the list.
 294                          */
 295                         if (pdp.isDesignated()  &&  !isCurrent) {
 296                             logger.debug("INFO: DesignatedWaiter.run: PDP= {} is currently "
 297                                             + "designated but is not current; "
 298                                             + "it has failed.  Standing down.  standbyStatus= {}",
 299                                             pdp.getPdpId(), standbyStatus);
 300                             /*
 301                              * Changes designated to 0 but it is still potentially providing service
 302                              * Will affect isDesignated, so, it can enter an if(combination) below
 303                              */
 304                             pdpsConnector.standDownPdp(pdp.getPdpId());
 305
 306                             //need to change standbystatus to coldstandby
 307                             if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 308                                 logger.debug("\n\nDesignatedWaiter.run: myPdp {} is not Current. "
 309                                                 + " Executing stateManagement.disableFailed()\n\n", myPdp.getPdpId());
 310                                 // We found that myPdp is designated but not current
 311                                 // So, we must cause it to disableFail
 312                                 try {
 313                                     myPdp.setDesignated(false);
 314                                     pdpsConnector.setDesignated(myPdp, false);
 315                                     isDesignated = false;
 316                                     stateManagementFeature.disableFailed();
 317                                 } catch (Exception e) {
 318                                     logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception "
 319                                             + "attempting to disableFail myPdp {}, message= {}",
 320                                             myPdp.getPdpId(), myPdp.getPdpId(), e);
 321                                 }
 322                             } else { //it is a remote PDP that is failed
 323                                 logger.debug("\n\nDesignatedWaiter.run: PDP {} is not Current. "
 324                                                 + " Executing stateManagement.disableFailed(otherResourceName)\n\n",
 325                                                 pdp.getPdpId() );
 326                                 // We found a PDP is designated but not current
 327                                 // We already called standdown(pdp) which will change designated to false
 328                                 // Now we need to disableFail it to get its states in synch.  The standbyStatus
 329                                 // should equal coldstandby
 330                                 try {
 331                                     stateManagementFeature.disableFailed(pdp.getPdpId());
 332                                 } catch (Exception e) {
 333                                     logger.error("DesignatedWaiter.run: for PDP {}  Caught Exception attempting to "
 334                                             + "disableFail({}), message= {}",
 335                                             pdp.getPdpId(), pdp.getPdpId(), e);
 336                                 }
 337
 338                             }
 339                             continue; //we are not going to do anything else with this pdp
 340                         }
 341
 342                         /*
 343                          * The third combination of isDesignated and isCurrent
 344                          * /*
 345                          * If a PDP is not currently designated but is providing service
 346                          * (erroneous, but recoverable) or hot standby
 347                          * we can add it to the list of possible designated if all the designated have failed
 348                          */
 349                         if (!pdp.isDesignated() && isCurrent) {
 350                             if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 351                                     || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 352                                 logger.debug("\n\nDesignatedWaiter.run: PDP {}"
 353                                                 + " is NOT designated but IS current and"
 354                                                 + " has a standbystatus= {}", pdp.getPdpId(), standbyStatus);
 355                                 // Since it is current, we assume it can adjust its own state.
 356                                 // We will demote if it is myPdp
 357                                 if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 358                                     //demote it
 359                                     logger.debug("DesignatedWaiter.run: PDP {} going to "
 360                                                     + "setDesignated = false and calling stateManagement.demote",
 361                                                     pdp.getPdpId());
 362                                     try {
 363                                         //Keep the order like this.
 364                                         //StateManagement is last since it triggers controller shutdown
 365                                         pdpsConnector.setDesignated(myPdp, false);
 366                                         myPdp.setDesignated(false);
 367                                         isDesignated = false;
 368                                         //This is definitely not a redundant call.
 369                                         //It is attempting to correct a problem
 370                                         stateManagementFeature.demote();
 371                                         //recheck the standbystatus
 372                                         standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 373                                     } catch (Exception e) {
 374                                         logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception "
 375                                                 + "attempting to demote myPdp {}, message = {}",  myPdp.getPdpId(),
 376                                                 myPdp.getPdpId(), e);
 377                                     }
 378
 379                                 }
 380                             }
 381                             if (standbyStatus.equals(StateManagement.HOT_STANDBY) && designatedPdpHasFailed) {
 382                                 //add it to the list
 383                                 logger.debug("INFO: DesignatedWaiter.run: PDP= {}"
 384                                                 + " is not designated but is {} and designated PDP "
 385                                                 + "has failed.  standbyStatus= {}", pdp.getPdpId(),
 386                                                 standbyStatus, standbyStatus);
 387                                 listOfDesignated.add(pdp);
 388                             }
 389                             continue; //done with this one
 390                         }
 391
 392                         /*
 393                          * The fourth combination of isDesignated and isCurrent
 394                          *
 395                          * We are not going to put any of these on the list since it appears they have failed.
 396
 397                          *
 398                          */
 399                         if (!pdp.isDesignated() && !isCurrent) {
 400                             logger.debug("INFO: DesignatedWaiter.run: PDP= {} "
 401                                             + "designated= {}, current= {}, "
 402                                             + "designatedPdpHasFailed= {}, "
 403                                             + "standbyStatus= {}",pdp.getPdpId(),
 404                                             pdp.isDesignated(), isCurrent, designatedPdpHasFailed, standbyStatus);
 405                             if (!standbyStatus.equals(StateManagement.COLD_STANDBY)) {
 406                                 //stand it down
 407                                 //disableFail it
 408                                 pdpsConnector.standDownPdp(pdp.getPdpId());
 409                                 if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 410                                     /*
 411                                      * I don't actually know how this condition could
 412                                      * happen, but if it did, we would want to declare it
 413                                      * failed.
 414                                      */
 415                                     logger.debug("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 416                                                     + " Executing stateManagement.disableFailed()\n\n",
 417                                                     myPdp.getPdpId());
 418                                     // So, we must disableFail it
 419                                     try {
 420                                         //Keep the order like this.
 421                                         //StateManagement is last since it triggers controller shutdown
 422                                         pdpsConnector.setDesignated(myPdp, false);
 423                                         myPdp.setDesignated(false);
 424                                         isDesignated = false;
 425                                         stateManagementFeature.disableFailed();
 426                                     } catch (Exception e) {
 427                                         logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 428                                                 + "disableFail myPdp {}, message= {}",
 429                                                 myPdp.getPdpId(), myPdp.getPdpId(), e);
 430                                     }
 431                                 } else { //it is remote
 432                                     logger.debug("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 433                                                     + " Executing stateManagement.disableFailed({})\n\n",
 434                                                     myPdp.getPdpId(), pdp.getPdpId());
 435                                     // We already called standdown(pdp) which will change designated to false
 436                                     // Now we need to disableFail it to get its states in sync.
 437                                     // StandbyStatus = coldstandby
 438                                     try {
 439                                         stateManagementFeature.disableFailed(pdp.getPdpId());
 440                                     } catch (Exception e) {
 441                                         logger.error("DesignatedWaiter.run: for PDP {}"
 442                                                 + " Caught Exception attempting to disableFail({})"
 443                                                 + ", message=", pdp.getPdpId(), pdp.getPdpId(), e);
 444                                     }
 445                                 }
 446                             }
 447                         }
 448
 449
 450                     } // end pdps loop
 451
 452                     /*
 453                      * We have checked the four combinations of isDesignated and isCurrent.  Where appropriate,
 454                      * we added the PDPs to the potential list of designated pdps
 455                      *
 456                      * We need to give priority to pdps on the same site that is currently being used
 457                      * First, however, we must sanitize the list of designated to make sure their are
 458                      * only designated members or non-designated members.  There should not be both in
 459                      * the list. Because there are real time delays, it is possible that both types could
 460                      * be on the list.
 461                      */
 462
 463                     listOfDesignated = santizeDesignatedList(listOfDesignated);
 464
 465                     /*
 466                      * We need to figure out the last pdp that was the primary so we can get the last site
 467                      * name and the last session numbers.  We need to create a "dummy" droolspdp since
 468                      * it will be used in later comparisons and cannot be null.
 469                      */
 470
 471                     DroolsPdp mostRecentPrimary = computeMostRecentPrimary(pdps, listOfDesignated);
 472
 473                     if (mostRecentPrimary != null) {
 474                         pdpdLastActive = mostRecentPrimary.getPdpId();
 475                     }
 476
 477
 478                     /*
 479                      * It is possible to get here with more than one pdp designated and providingservice. This normally
 480                      * occurs when there is a race condition with multiple nodes coming up at the same time. If that is
 481                      * the case we must determine which one is the one that should be designated and which one should
 482                      * be demoted.
 483                      *
 484                      * It is possible to have 0, 1, 2 or more but not all, or all designated.
 485                      *   If we have one designated and current, we chose it and are done
 486                      *   If we have 2 or more, but not all, we must determine which one is in the same site as
 487                      *   the previously designated pdp.
 488                      */
 489                     DroolsPdp designatedPdp = computeDesignatedPdp(listOfDesignated, mostRecentPrimary);
 490                     if (designatedPdp != null) {
 491                         pdpdNowActive = designatedPdp.getPdpId();
 492                     }
 493
 494                     if (designatedPdp == null) {
 495                         logger.warn("WARNING: DesignatedWaiter.run: No viable PDP found to be Designated. "
 496                             + "designatedPdp still null.");
 497                         // Just to be sure the parameters are correctly set
 498                         myPdp.setDesignated(false);
 499                         pdpsConnector.setDesignated(myPdp,false);
 500                         isDesignated = false;
 501
 502                         waitTimerLastRunDate = new Date();
 503                         logger.debug("DesignatedWaiter.run (designatedPdp == null) waitTimerLastRunDate = {}",
 504                                         waitTimerLastRunDate);
 505                         myPdp.setUpdatedDate(waitTimerLastRunDate);
 506                         pdpsConnector.update(myPdp);
 507
 508                         return;
 509
 510                     } else if (designatedPdp.getPdpId().equals(myPdp.getPdpId())) {
 511                         logger.debug("DesignatedWaiter.run: designatedPdp is PDP={}", myPdp.getPdpId());
 512                         /*
 513                          * update function expects myPdp.isDesignated to be true.
 514                          */
 515                         try {
 516                             //Keep the order like this.  StateManagement is last since it triggers controller init
 517                             myPdp.setDesignated(true);
 518                             myPdp.setDesignatedDate(new Date());
 519                             pdpsConnector.setDesignated(myPdp, true);
 520                             isDesignated = true;
 521                             String standbyStatus = stateManagementFeature.getStandbyStatus();
 522                             if (!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)) {
 523                                 /*
 524                                  * Only call promote if it is not already in the right state.  Don't worry about
 525                                  * synching the lower level topic endpoint states.  That is done by the
 526                                  * refreshStateAudit.
 527                                  * Note that we need to fetch the session list from 'mostRecentPrimary'
 528                                  * at this point -- soon, 'mostRecentPrimary' will be set to this host.
 529                                  */
 530                                 //this.sessions = mostRecentPrimary.getSessions();
 531                                 stateManagementFeature.promote();
 532                             }
 533                         } catch (Exception e) {
 534                             logger.error("ERROR: DesignatedWaiter.run: Caught Exception attempting to promote PDP={}"
 535                                     + ", message=", myPdp.getPdpId(), e);
 536                             myPdp.setDesignated(false);
 537                             pdpsConnector.setDesignated(myPdp,false);
 538                             isDesignated = false;
 539                             //If you can't promote it, demote it
 540                             try {
 541                                 String standbyStatus = stateManagementFeature.getStandbyStatus();
 542                                 if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 543                                         || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 544                                     /*
 545                                      * Only call demote if it is not already in the right state.  Don't worry about
 546                                      * synching the lower level topic endpoint states.  That is done by the
 547                                      * refreshStateAudit.
 548                                      */
 549                                     stateManagementFeature.demote();
 550                                 }
 551                             } catch (Exception e1) {
 552                                 logger.error("ERROR: DesignatedWaiter.run: Caught StandbyStatusException "
 553                                         + "attempting to promote then demote PDP={}, message=",
 554                                         myPdp.getPdpId(), e1);
 555                             }
 556
 557                         }
 558                         waitTimerLastRunDate = new Date();
 559                         logger.debug("DesignatedWaiter.run (designatedPdp.getPdpId().equals(myPdp.getPdpId())) "
 560                                         + "waitTimerLastRunDate = " + waitTimerLastRunDate);
 561                         myPdp.setUpdatedDate(waitTimerLastRunDate);
 562                         pdpsConnector.update(myPdp);
 563
 564                         return;
 565                     }
 566                     isDesignated = false;
 567
 568                 } // end synchronized
 569                 logger.debug("DesignatedWaiter.run: myPdp: {}; Returning, isDesignated= {}",
 570                                 isDesignated, myPdp.getPdpId());
 571
 572                 Date tmpDate = new Date();
 573                 logger.debug("DesignatedWaiter.run (end of run) waitTimerLastRunDate = {}", tmpDate);
 574
 575                 waitTimerLastRunDate = tmpDate;
 576                 myPdp.setUpdatedDate(waitTimerLastRunDate);
 577                 pdpsConnector.update(myPdp);
 578
 579             } catch (Exception e) {
 580                 logger.error("DesignatedWaiter.run caught an unexpected exception: ", e);
 581             }
 582         } // end run
 583     }
 584
 585     /**
 586      * Sanitize designated list.
 587      *
 588      * @param listOfDesignated list of designated pdps
 589      * @return list of drools pdps
 590      */
 591     public List<DroolsPdp> santizeDesignatedList(List<DroolsPdp> listOfDesignated) {
 592
 593         boolean containsDesignated = false;
 594         boolean containsHotStandby = false;
 595         List<DroolsPdp> listForRemoval = new ArrayList<>();
 596         for (DroolsPdp pdp : listOfDesignated) {
 597             logger.debug("DesignatedWaiter.run sanitizing: pdp = {}"
 598                             + " isDesignated = {}",pdp.getPdpId(), pdp.isDesignated());
 599             if (pdp.isDesignated()) {
 600                 containsDesignated = true;
 601             } else {
 602                 containsHotStandby = true;
 603                 listForRemoval.add(pdp);
 604             }
 605         }
 606         if (containsDesignated && containsHotStandby) {
 607             //remove the hot standby from the list
 608             listOfDesignated.removeAll(listForRemoval);
 609         }
 610         return listOfDesignated;
 611     }
 612
 613     /**
 614      * Compute most recent primary.
 615      *
 616      * @param pdps collection of pdps
 617      * @param listOfDesignated list of designated pdps
 618      * @return drools pdp object
 619      */
 620     public DroolsPdp computeMostRecentPrimary(Collection<DroolsPdp> pdps, List<DroolsPdp> listOfDesignated) {
 621         boolean containsDesignated = false;
 622         for (DroolsPdp pdp : listOfDesignated) {
 623             if (pdp.isDesignated()) {
 624                 containsDesignated = true;
 625             }
 626         }
 627         DroolsPdp mostRecentPrimary = new DroolsPdpImpl(null, true, 1, new Date(0));
 628         mostRecentPrimary.setSiteName(null);
 629         logger.debug("DesignatedWaiter.run listOfDesignated.size() = {}", listOfDesignated.size());
 630         if (listOfDesignated.size() <= 1) {
 631             logger.debug("DesignatedWainter.run: listOfDesignated.size <=1");
 632             //Only one or none is designated or hot standby.  Choose the latest designated date
 633             for (DroolsPdp pdp : pdps) {
 634                 logger.debug("DesignatedWaiter.run pdp = {}"
 635                                 + " pdp.getDesignatedDate() = {}",
 636                                 pdp.getPdpId(), pdp.getDesignatedDate());
 637                 if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 638                     mostRecentPrimary = pdp;
 639                     logger.debug("DesignatedWaiter.run mostRecentPrimary = {}",
 640                                     mostRecentPrimary.getPdpId());
 641                 }
 642             }
 643         } else if (listOfDesignated.size() == pdps.size()) {
 644             logger.debug("DesignatedWainter.run: listOfDesignated.size = pdps.size() which is {}", pdps.size());
 645             //They are all designated or all hot standby.
 646             mostRecentPrimary = null;
 647             for (DroolsPdp pdp : pdps) {
 648                 if (mostRecentPrimary == null) {
 649                     mostRecentPrimary = pdp;
 650                     continue;
 651                 }
 652                 if (containsDesignated) { //Choose the site of the first designated date
 653                     if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) < 0) {
 654                         mostRecentPrimary = pdp;
 655                         logger.debug("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 656                     }
 657                 } else { //Choose the site with the latest designated date
 658                     if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 659                         mostRecentPrimary = pdp;
 660                         logger.debug("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 661                     }
 662                 }
 663             }
 664         } else {
 665             logger.debug("DesignatedWainter.run: Some but not all are designated or hot standby. ");
 666             //Some but not all are designated or hot standby.
 667             if (containsDesignated) {
 668                 logger.debug("DesignatedWainter.run: containsDesignated = {}", containsDesignated);
 669                 /*
 670                  * The list only contains designated.  This is a problem.  It is most likely a race
 671                  * condition that resulted in two thinking they should be designated. Choose the
 672                  * site with the latest designated date for the pdp not included on the designated list.
 673                  * This should be the site that had the last designation before this race condition
 674                  * occurred.
 675                  */
 676                 for (DroolsPdp pdp : pdps) {
 677                     if (listOfDesignated.contains(pdp)) {
 678                         continue; //Don't consider this entry
 679                     }
 680                     if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 681                         mostRecentPrimary = pdp;
 682                         logger.debug("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 683                     }
 684                 }
 685             } else {
 686                 logger.debug("DesignatedWainter.run: containsDesignated = {}", containsDesignated);
 687                 //The list only contains hot standby. Choose the site of the latest designated date
 688                 for (DroolsPdp pdp : pdps) {
 689                     if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 690                         mostRecentPrimary = pdp;
 691                         logger.debug("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 692                     }
 693                 }
 694             }
 695         }
 696         return mostRecentPrimary;
 697     }
 698
 699     /**
 700      * Compue designated pdp.
 701      *
 702      * @param listOfDesignated list of designated pdps
 703      * @param mostRecentPrimary most recent primary pdpd
 704      * @return drools pdp object
 705      */
 706     public DroolsPdp computeDesignatedPdp(List<DroolsPdp> listOfDesignated, DroolsPdp mostRecentPrimary) {
 707         DroolsPdp designatedPdp = null;
 708         DroolsPdp lowestPriorityPdp = null;
 709         if (listOfDesignated.size() > 1) {
 710             logger.debug("DesignatedWaiter.run: myPdp: {} listOfDesignated.size(): {}", myPdp.getPdpId(),
 711                             listOfDesignated.size());
 712             DroolsPdp rejectedPdp = null;
 713             DroolsPdp lowestPrioritySameSite = null;
 714             DroolsPdp lowestPriorityDifferentSite = null;
 715             for (DroolsPdp pdp : listOfDesignated) {
 716                 // We need to determine if another PDP is the lowest priority
 717                 if (nullSafeEquals(pdp.getSiteName(),mostRecentPrimary.getSiteName())) {
 718                     if (lowestPrioritySameSite == null) {
 719                         if (lowestPriorityDifferentSite != null) {
 720                             rejectedPdp = lowestPriorityDifferentSite;
 721                         }
 722                         lowestPrioritySameSite = pdp;
 723                     } else {
 724                         if (pdp.getPdpId().equals((lowestPrioritySameSite.getPdpId()))) {
 725                             continue;//nothing to compare
 726                         }
 727                         if (pdp.comparePriority(lowestPrioritySameSite) < 0) {
 728                             logger.debug("\nDesignatedWaiter.run: myPdp {}  listOfDesignated pdp ID: {}"
 729                                             + " has lower priority than pdp ID: {}",myPdp.getPdpId(), pdp.getPdpId(),
 730                                             lowestPrioritySameSite.getPdpId());
 731                             //we need to reject lowestPrioritySameSite
 732                             rejectedPdp = lowestPrioritySameSite;
 733                             lowestPrioritySameSite = pdp;
 734                         } else {
 735                             //we need to reject pdp and keep lowestPrioritySameSite
 736                             logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {} "
 737                                             + " has higher priority than pdp ID: {}", myPdp.getPdpId(),pdp.getPdpId(),
 738                                             lowestPrioritySameSite.getPdpId());
 739                             rejectedPdp = pdp;
 740                         }
 741                     }
 742                 } else {
 743                     if (lowestPrioritySameSite != null) {
 744                         //if we already have a candidate for same site, we don't want to bother with different sites
 745                         rejectedPdp = pdp;
 746                     } else {
 747                         if (lowestPriorityDifferentSite == null) {
 748                             lowestPriorityDifferentSite = pdp;
 749                             continue;
 750                         }
 751                         if (pdp.getPdpId().equals((lowestPriorityDifferentSite.getPdpId()))) {
 752                             continue;//nothing to compare
 753                         }
 754                         if (pdp.comparePriority(lowestPriorityDifferentSite) < 0) {
 755                             logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 756                                             + " has lower priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 757                                             lowestPriorityDifferentSite.getPdpId());
 758                             //we need to reject lowestPriorityDifferentSite
 759                             rejectedPdp = lowestPriorityDifferentSite;
 760                             lowestPriorityDifferentSite = pdp;
 761                         } else {
 762                             //we need to reject pdp and keep lowestPriorityDifferentSite
 763                             logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 764                                             + " has higher priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 765                                             lowestPriorityDifferentSite.getPdpId());
 766                             rejectedPdp = pdp;
 767                         }
 768                     }
 769                 }
 770                 // If the rejectedPdp is myPdp, we need to stand it down and demote it.  Each pdp is responsible
 771                 // for demoting itself
 772                 if (rejectedPdp != null && nullSafeEquals(rejectedPdp.getPdpId(),myPdp.getPdpId())) {
 773                     logger.debug("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated myPdp ID: {}"
 774                                     + " is NOT the lowest priority.  Executing stateManagement.demote()\n\n",
 775                                     myPdp.getPdpId(),
 776                                     myPdp.getPdpId());
 777                     // We found that myPdp is on the listOfDesignated and it is not the lowest priority
 778                     // So, we must demote it
 779                     try {
 780                         //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 781                         myPdp.setDesignated(false);
 782                         pdpsConnector.setDesignated(myPdp, false);
 783                         isDesignated = false;
 784                         String standbyStatus = stateManagementFeature.getStandbyStatus();
 785                         if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 786                                 || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 787                             /*
 788                              * Only call demote if it is not already in the right state.  Don't worry about
 789                              * synching the lower level topic endpoint states.  That is done by the
 790                              * refreshStateAudit.
 791                              */
 792                             stateManagementFeature.demote();
 793                         }
 794                     } catch (Exception e) {
 795                         myPdp.setDesignated(false);
 796                         pdpsConnector.setDesignated(myPdp, false);
 797                         isDesignated = false;
 798                         logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 799                                 + "demote myPdp {} myPdp.getPdpId(), message= {}", myPdp.getPdpId(),
 800                                 e);
 801                     }
 802                 }
 803             } //end: for(DroolsPdp pdp : listOfDesignated)
 804             if (lowestPrioritySameSite != null) {
 805                 lowestPriorityPdp = lowestPrioritySameSite;
 806             } else {
 807                 lowestPriorityPdp = lowestPriorityDifferentSite;
 808             }
 809             //now we have a valid value for lowestPriorityPdp
 810             logger.debug("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 811                             + "found the LOWEST priority pdp ID: {} "
 812                             + " It is now the designatedPpd from the perspective of myPdp ID: {} \n\n",
 813                             myPdp.getPdpId(), lowestPriorityPdp.getPdpId(), myPdp);
 814             designatedPdp = lowestPriorityPdp;
 815
 816         } else if (listOfDesignated.isEmpty()) {
 817             logger.debug("\nDesignatedWaiter.run: myPdp: {} listOfDesignated is: EMPTY.", myPdp.getPdpId());
 818             designatedPdp = null;
 819         } else { //only one in listOfDesignated
 820             logger.debug("\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 821                             + "has ONE entry. PDP ID: {}", myPdp.getPdpId(), listOfDesignated.get(0).getPdpId());
 822             designatedPdp = listOfDesignated.get(0);
 823         }
 824         return designatedPdp;
 825
 826     }
 827
 828     private class TimerUpdateClass extends TimerTask {
 829
 830         @Override
 831         public void run() {
 832             try {
 833                 logger.debug("TimerUpdateClass.run: entry");
 834                 checkWaitTimer();
 835             } catch (Exception e) {
 836                 logger.error("TimerUpdateClass.run caught an unexpected exception: ", e);
 837             }
 838             logger.debug("TimerUpdateClass.run.exit");
 839         }
 840     }
 841
 842     @Override
 843     public void checkThreadStatus() {
 844         checkWaitTimer();
 845     }
 846
 847     private void checkWaitTimer() {
 848         synchronized (checkWaitTimerLock) {
 849             try {
 850                 logger.debug("checkWaitTimer: entry");
 851                 Date now = new Date();
 852                 long nowMs = now.getTime();
 853                 long waitTimerMs = waitTimerLastRunDate.getTime();
 854
 855                 //give it 10 times leeway
 856                 if ((nowMs - waitTimerMs)  > 10 * pdpUpdateInterval) {
 857                     if (allSeemsWell == null || allSeemsWell) {
 858                         allSeemsWell = false;
 859                         logger.debug("checkWaitTimer: calling allSeemsWell with ALLNOTWELL param");
 860                         stateManagementFeature.allSeemsWell(this.getClass().getName(),
 861                                 StateManagementFeatureApi.ALLNOTWELL_STATE,
 862                                 "DesignationWaiter/ElectionHandler has STALLED");
 863                     }
 864                     logger.error("checkWaitTimer: nowMs - waitTimerMs = {}"
 865                             + ", exceeds 10* pdpUpdateInterval = {}"
 866                             + " DesignationWaiter is STALLED!", (nowMs - waitTimerMs), (10 * pdpUpdateInterval));
 867                 } else if (allSeemsWell == null || !allSeemsWell) {
 868                     allSeemsWell = true;
 869                     stateManagementFeature.allSeemsWell(this.getClass().getName(),
 870                             StateManagementFeatureApi.ALLSEEMSWELL_STATE,
 871                             "DesignationWaiter/ElectionHandler has RESUMED");
 872                     logger.info("DesignationWaiter/ElectionHandler has RESUMED");
 873                 }
 874                 logger.debug("checkWaitTimer: exit");
 875             } catch (Exception e) {
 876                 logger.error("checkWaitTimer: caught unexpected exception: ", e);
 877             }
 878         }
 879     }
 880
 881     private long getDWaiterStartMs() {
 882         Date now = new Date();
 883
 884         // Retrieve the ms since the epoch
 885         long nowMs = now.getTime();
 886
 887         // Time since the end of the last pdpUpdateInterval multiple
 888         long nowModMs = nowMs % pdpUpdateInterval;
 889
 890         // Time to the start of the next pdpUpdateInterval multiple
 891         long startMs = 2 * pdpUpdateInterval - nowModMs;
 892
 893         // Give the start time a minimum of a 5 second cushion
 894         if (startMs < 5000) {
 895             // Start at the beginning  of following interval
 896             startMs = pdpUpdateInterval + startMs;
 897         }
 898         return startMs;
 899     }
 900
 901     private boolean nullSafeEquals(Object one, Object two) {
 902         if (one == null && two == null) {
 903             return true;
 904         }
 905         if (one != null && two != null) {
 906             return one.equals(two);
 907         }
 908         return false;
 909     }
 910
 911     public String getPdpdNowActive() {
 912         return pdpdNowActive;
 913     }
 914
 915     public String getPdpdLastActive() {
 916         return pdpdLastActive;
 917     }
 918 }