gerrit.onap Code Review - policy/drools-pdp.git/blob

   1 /*
   2  * ============LICENSE_START=======================================================
   3  * feature-active-standby-management
   4  * ================================================================================
   5  * Copyright (C) 2017-2019 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.policy.drools.activestandby;
  22
  23 import java.util.ArrayList;
  24 import java.util.Collection;
  25 import java.util.Date;
  26 import java.util.List;
  27 import java.util.Timer;
  28 import java.util.TimerTask;
  29
  30 import org.onap.policy.common.im.StateManagement;
  31 import org.onap.policy.drools.statemanagement.StateManagementFeatureApi;
  32 import org.onap.policy.drools.statemanagement.StateManagementFeatureApiConstants;
  33 import org.slf4j.Logger;
  34 import org.slf4j.LoggerFactory;
  35
  36 public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
  37     private static final String RUN_PRIMARY_MSG = "DesignatedWaiter.run mostRecentPrimary = {}";
  38
  39     // get an instance of logger
  40     private static final Logger  logger = LoggerFactory.getLogger(DroolsPdpsElectionHandler.class);
  41     private DroolsPdpsConnector pdpsConnector;
  42     private Object checkWaitTimerLock = new Object();
  43     private Object designationWaiterLock = new Object();
  44
  45     /*
  46      * Must be static, so it can be referenced by JpaDroolsPdpsConnector,
  47      * without requiring a reference to the election handler instantiation.
  48      */
  49     private static DroolsPdp myPdp;
  50
  51     private DesignationWaiter designationWaiter;
  52     private Timer updateWorker;
  53     private Timer waitTimer;
  54     private Date waitTimerLastRunDate;
  55
  56     // The interval between checks of the DesignationWaiter to be sure it is running.
  57     private int pdpCheckInterval;
  58
  59     // The interval between runs of the DesignationWaiter
  60     private int pdpUpdateInterval;
  61
  62     private volatile boolean isDesignated;
  63
  64     private String pdpdNowActive;
  65     private String pdpdLastActive;
  66
  67     /*
  68      * Start allSeemsWell with a value of null so that, on the first run
  69      * of the checkWaitTimer it will set the value in IntegrityMonitor
  70      * regardless of whether it needs to be set to true or false.
  71      */
  72     private Boolean allSeemsWell = null;
  73
  74     private StateManagementFeatureApi stateManagementFeature;
  75
  76     private static boolean isUnitTesting = false;
  77     private static boolean isStalled = false;
  78
  79     /**
  80      * Constructor.
  81      *
  82      * @param pdps connectors
  83      * @param myPdp pdp
  84      */
  85     public DroolsPdpsElectionHandler(DroolsPdpsConnector pdps, DroolsPdp myPdp) {
  86         if (pdps == null) {
  87             logger.error("DroolsPdpsElectinHandler(): pdpsConnector==null");
  88             throw new IllegalArgumentException("DroolsPdpsElectinHandler(): pdpsConnector==null");
  89         }
  90         if (myPdp == null) {
  91             logger.error("DroolsPdpsElectinHandler(): droolsPdp==null");
  92             throw new IllegalArgumentException("DroolsPdpsElectinHandler(): DroolsPdp==null");
  93         }
  94
  95         pdpdNowActive = null;
  96         pdpdLastActive = null;
  97         this.pdpsConnector = pdps;
  98         DroolsPdpsElectionHandler.myPdp = myPdp;
  99         this.isDesignated = false;
 100         pdpCheckInterval = 3000;
 101         try {
 102             pdpCheckInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(
 103                     ActiveStandbyProperties.PDP_CHECK_INVERVAL));
 104         } catch (Exception e) {
 105             logger.error("Could not get pdpCheckInterval property. Using default {}",pdpCheckInterval, e);
 106         }
 107         pdpUpdateInterval = 2000;
 108         try {
 109             pdpUpdateInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(
 110                     ActiveStandbyProperties.PDP_UPDATE_INTERVAL));
 111         } catch (Exception e) {
 112             logger.error("Could not get pdpUpdateInterval property. Using default {} ", pdpUpdateInterval, e);
 113         }
 114
 115         Date now = new Date();
 116
 117         // Retrieve the ms since the epoch
 118         final long nowMs = now.getTime();
 119
 120         // Create the timer which will update the updateDate in DroolsPdpEntity table.
 121         // This is the heartbeat
 122         updateWorker = new Timer();
 123
 124         // Schedule the TimerUpdateClass to run at 100 ms and run at pdpCheckInterval ms thereafter
 125         // NOTE: The first run of the TimerUpdateClass results in myPdp being added to the
 126         // drools droolsPdpEntity table.
 127         updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
 128
 129         // Create the timer which will run the election algorithm
 130         waitTimer = new Timer();
 131
 132         // Schedule it to start in startMs ms
 133         // (so it will run after the updateWorker and run at pdpUpdateInterval ms thereafter
 134         long startMs = getDWaiterStartMs();
 135         designationWaiter = new DesignationWaiter();
 136         waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
 137         waitTimerLastRunDate = new Date(nowMs + startMs);
 138
 139         //Get the StateManagementFeature instance
 140
 141         for (StateManagementFeatureApi feature : StateManagementFeatureApiConstants.getImpl().getList()) {
 142             if (feature.getResourceName().equals(myPdp.getPdpId())) {
 143                 logger.debug("DroolsPdpsElectionHandler: Found StateManagementFeature"
 144                                 + " with resourceName: {}", myPdp.getPdpId());
 145                 stateManagementFeature = feature;
 146                 break;
 147             }
 148         }
 149         if (stateManagementFeature == null) {
 150             logger.error("DroolsPdpsElectionHandler failed to initialize.  "
 151                     + "Unable to get instance of StateManagementFeatureApi "
 152                     + "with resourceID: {}", myPdp.getPdpId());
 153         }
 154     }
 155
 156     public static void setIsUnitTesting(boolean val) {
 157         isUnitTesting = val;
 158     }
 159
 160     public static void setIsStalled(boolean val) {
 161         isStalled = val;
 162     }
 163
 164     /**
 165      * When the JpaDroolsPdpsConnector.standDown() method is invoked, it needs
 166      * access to myPdp, so it can keep its designation status in sync with the
 167      * DB.
 168      *
 169      * @param designated is designated value
 170      */
 171     public static void setMyPdpDesignated(boolean designated) {
 172         logger.debug("setMyPdpDesignated: designated= {}", designated);
 173         myPdp.setDesignated(designated);
 174     }
 175
 176     private class DesignationWaiter extends TimerTask {
 177         // get an instance of logger
 178         private final Logger  logger = LoggerFactory.getLogger(DesignationWaiter.class);
 179
 180         @Override
 181         public void run() {
 182             try {
 183                 logger.debug("DesignatedWaiter.run: Entering");
 184
 185                 //This is for testing the checkWaitTimer
 186                 if (isUnitTesting && isStalled) {
 187                     logger.debug("DesignatedWaiter.run: isUnitTesting = {} isStalled = {}",
 188                                     isUnitTesting, isStalled);
 189                     return;
 190                 }
 191
 192                 synchronized (designationWaiterLock) {
 193
 194                     logger.debug("DesignatedWaiter.run: Entering synchronized block");
 195
 196                     //It is possible that multiple PDPs are designated lead.  So, we will make a list of all designated
 197                     //PDPs and then decide which one really should be designated at the end.
 198                     List<DroolsPdp> listOfDesignated = new ArrayList<>();
 199
 200                     Collection<DroolsPdp> pdps = pdpsConnector.getDroolsPdps();
 201
 202                     logger.debug("DesignatedWaiter.run: pdps.size= {}", pdps.size());
 203
 204                     //This is only true if all designated PDPs have failed
 205                     boolean designatedPdpHasFailed = pdpsConnector.hasDesignatedPdpFailed(pdps);
 206                     logger.debug("DesignatedWaiter.run: designatedPdpHasFailed= {}", designatedPdpHasFailed);
 207                     for (DroolsPdp pdp : pdps) {
 208                         logger.debug("DesignatedWaiter.run: evaluating pdp ID: {}", pdp.getPdpId());
 209
 210                         /*
 211                          * Note: side effect of isPdpCurrent is that any stale but
 212                          * designated PDPs will be marked as un-designated.
 213                          */
 214                         boolean isCurrent = pdpsConnector.isPdpCurrent(pdp);
 215
 216                         /*
 217                          * We can't use stateManagement.getStandbyStatus() here, because
 218                          * we need the standbyStatus, not for this PDP, but for the PDP
 219                          * being processed by this loop iteration.
 220                          */
 221                         String standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 222                         if (standbyStatus == null) {
 223                             // Treat this case as a cold standby -- if we
 224                             // abort here, no sessions will be created in a
 225                             // single-node test environment.
 226                             standbyStatus = StateManagement.COLD_STANDBY;
 227                         }
 228                         logger.debug("DesignatedWaiter.run: PDP= {},  isCurrent= {}", pdp.getPdpId(), isCurrent);
 229
 230                         /*
 231                          * There are 4 combinations of isDesignated and isCurrent.  We will examine each one in-turn
 232                          * and evaluate the each pdp in the list of pdps against each combination.
 233                          *
 234                          * This is the first combination of isDesignated and isCurrent
 235                          */
 236                         if (pdp.isDesignated()  &&  isCurrent) {
 237                             //It is current, but it could have a standbystatus=coldstandby / hotstandby
 238                             //If so, we need to stand it down and demote it
 239                             if (!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)) {
 240                                 if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 241                                     logger.debug("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 242                                                     + "butstandbystatus is not providingservice. "
 243                                                     + " Executing stateManagement.demote()" + "\n\n", myPdp.getPdpId());
 244                                     // So, we must demote it
 245                                     try {
 246                                         //Keep the order like this.  StateManagement is last since it
 247                                         //triggers controller shutdown
 248                                         //This will change isDesignated and it can enter another if(combination) below
 249                                         pdpsConnector.standDownPdp(pdp.getPdpId());
 250                                         myPdp.setDesignated(false);
 251                                         isDesignated = false;
 252                                         if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 253                                                 || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 254                                             /*
 255                                              * Only demote it if it appears it has not already been demoted. Don't worry
 256                                              * about synching with the topic endpoint states.  That is done by the
 257                                              * refreshStateAudit
 258                                              */
 259                                             stateManagementFeature.demote();
 260                                         }
 261                                         //update the standbystatus to check in a later
 262                                         //combination of isDesignated and isCurrent
 263                                         standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 264                                     } catch (Exception e) {
 265                                         logger.error("DesignatedWaiter.run: myPdp: {} "
 266                                                 + "Caught Exception attempting to demote myPdp,"
 267                                                 + "message= {}", myPdp.getPdpId(), e);
 268                                     }
 269                                 } else {
 270                                     // Don't demote a remote PDP that is current.  It should catch itself
 271                                     logger.debug("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 272                                                     + "but standbystatus is not providingservice. "
 273                                                     + " Cannot execute stateManagement.demote() "
 274                                                     + "since it it is not myPdp\n\n",
 275                                                     myPdp.getPdpId());
 276                                 }
 277
 278                             } else {
 279                                 // If we get here, it is ok to be on the list
 280                                 logger.debug("DesignatedWaiter.run: PDP= {} is designated, "
 281                                                 + "current and {} Noting PDP as "
 282                                                 + "designated, standbyStatus= {}",
 283                                                 pdp.getPdpId(), standbyStatus, standbyStatus);
 284                                 listOfDesignated.add(pdp);
 285                             }
 286
 287
 288                         }
 289
 290
 291                         /*
 292                          * The second combination of isDesignated and isCurrent
 293                          *
 294                          * PDP is designated but not current; it has failed.
 295                          * So we stand it down (it doesn't matter what
 296                          * its standbyStatus is). None of these go on the list.
 297                          */
 298                         if (pdp.isDesignated()  &&  !isCurrent) {
 299                             logger.debug("INFO: DesignatedWaiter.run: PDP= {} is currently "
 300                                             + "designated but is not current; "
 301                                             + "it has failed.  Standing down.  standbyStatus= {}",
 302                                             pdp.getPdpId(), standbyStatus);
 303                             /*
 304                              * Changes designated to 0 but it is still potentially providing service
 305                              * Will affect isDesignated, so, it can enter an if(combination) below
 306                              */
 307                             pdpsConnector.standDownPdp(pdp.getPdpId());
 308
 309                             //need to change standbystatus to coldstandby
 310                             if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 311                                 logger.debug("\n\nDesignatedWaiter.run: myPdp {} is not Current. "
 312                                                 + " Executing stateManagement.disableFailed()\n\n", myPdp.getPdpId());
 313                                 // We found that myPdp is designated but not current
 314                                 // So, we must cause it to disableFail
 315                                 try {
 316                                     myPdp.setDesignated(false);
 317                                     pdpsConnector.setDesignated(myPdp, false);
 318                                     isDesignated = false;
 319                                     stateManagementFeature.disableFailed();
 320                                 } catch (Exception e) {
 321                                     logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception "
 322                                             + "attempting to disableFail myPdp {}, message= {}",
 323                                             myPdp.getPdpId(), myPdp.getPdpId(), e);
 324                                 }
 325                             } else { //it is a remote PDP that is failed
 326                                 logger.debug("\n\nDesignatedWaiter.run: PDP {} is not Current. "
 327                                                 + " Executing stateManagement.disableFailed(otherResourceName)\n\n",
 328                                                 pdp.getPdpId() );
 329                                 // We found a PDP is designated but not current
 330                                 // We already called standdown(pdp) which will change designated to false
 331                                 // Now we need to disableFail it to get its states in synch.  The standbyStatus
 332                                 // should equal coldstandby
 333                                 try {
 334                                     stateManagementFeature.disableFailed(pdp.getPdpId());
 335                                 } catch (Exception e) {
 336                                     logger.error("DesignatedWaiter.run: for PDP {}  Caught Exception attempting to "
 337                                             + "disableFail({}), message= {}",
 338                                             pdp.getPdpId(), pdp.getPdpId(), e);
 339                                 }
 340
 341                             }
 342                             continue; //we are not going to do anything else with this pdp
 343                         }
 344
 345                         /*
 346                          * The third combination of isDesignated and isCurrent
 347                          * /*
 348                          * If a PDP is not currently designated but is providing service
 349                          * (erroneous, but recoverable) or hot standby
 350                          * we can add it to the list of possible designated if all the designated have failed
 351                          */
 352                         if (!pdp.isDesignated() && isCurrent) {
 353                             if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 354                                     || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 355                                 logger.debug("\n\nDesignatedWaiter.run: PDP {}"
 356                                                 + " is NOT designated but IS current and"
 357                                                 + " has a standbystatus= {}", pdp.getPdpId(), standbyStatus);
 358                                 // Since it is current, we assume it can adjust its own state.
 359                                 // We will demote if it is myPdp
 360                                 if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 361                                     //demote it
 362                                     logger.debug("DesignatedWaiter.run: PDP {} going to "
 363                                                     + "setDesignated = false and calling stateManagement.demote",
 364                                                     pdp.getPdpId());
 365                                     try {
 366                                         //Keep the order like this.
 367                                         //StateManagement is last since it triggers controller shutdown
 368                                         pdpsConnector.setDesignated(myPdp, false);
 369                                         myPdp.setDesignated(false);
 370                                         isDesignated = false;
 371                                         //This is definitely not a redundant call.
 372                                         //It is attempting to correct a problem
 373                                         stateManagementFeature.demote();
 374                                         //recheck the standbystatus
 375                                         standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 376                                     } catch (Exception e) {
 377                                         logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception "
 378                                                 + "attempting to demote myPdp {}, message = {}",  myPdp.getPdpId(),
 379                                                 myPdp.getPdpId(), e);
 380                                     }
 381
 382                                 }
 383                             }
 384                             if (standbyStatus.equals(StateManagement.HOT_STANDBY) && designatedPdpHasFailed) {
 385                                 //add it to the list
 386                                 logger.debug("INFO: DesignatedWaiter.run: PDP= {}"
 387                                                 + " is not designated but is {} and designated PDP "
 388                                                 + "has failed.  standbyStatus= {}", pdp.getPdpId(),
 389                                                 standbyStatus, standbyStatus);
 390                                 listOfDesignated.add(pdp);
 391                             }
 392                             continue; //done with this one
 393                         }
 394
 395                         /*
 396                          * The fourth combination of isDesignated and isCurrent
 397                          *
 398                          * We are not going to put any of these on the list since it appears they have failed.
 399
 400                          *
 401                          */
 402                         if (!pdp.isDesignated() && !isCurrent) {
 403                             logger.debug("INFO: DesignatedWaiter.run: PDP= {} "
 404                                             + "designated= {}, current= {}, "
 405                                             + "designatedPdpHasFailed= {}, "
 406                                             + "standbyStatus= {}",pdp.getPdpId(),
 407                                             pdp.isDesignated(), isCurrent, designatedPdpHasFailed, standbyStatus);
 408                             if (!standbyStatus.equals(StateManagement.COLD_STANDBY)) {
 409                                 //stand it down
 410                                 //disableFail it
 411                                 pdpsConnector.standDownPdp(pdp.getPdpId());
 412                                 if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 413                                     /*
 414                                      * I don't actually know how this condition could
 415                                      * happen, but if it did, we would want to declare it
 416                                      * failed.
 417                                      */
 418                                     logger.debug("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 419                                                     + " Executing stateManagement.disableFailed()\n\n",
 420                                                     myPdp.getPdpId());
 421                                     // So, we must disableFail it
 422                                     try {
 423                                         //Keep the order like this.
 424                                         //StateManagement is last since it triggers controller shutdown
 425                                         pdpsConnector.setDesignated(myPdp, false);
 426                                         myPdp.setDesignated(false);
 427                                         isDesignated = false;
 428                                         stateManagementFeature.disableFailed();
 429                                     } catch (Exception e) {
 430                                         logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 431                                                 + "disableFail myPdp {}, message= {}",
 432                                                 myPdp.getPdpId(), myPdp.getPdpId(), e);
 433                                     }
 434                                 } else { //it is remote
 435                                     logger.debug("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 436                                                     + " Executing stateManagement.disableFailed({})\n\n",
 437                                                     myPdp.getPdpId(), pdp.getPdpId());
 438                                     // We already called standdown(pdp) which will change designated to false
 439                                     // Now we need to disableFail it to get its states in sync.
 440                                     // StandbyStatus = coldstandby
 441                                     try {
 442                                         stateManagementFeature.disableFailed(pdp.getPdpId());
 443                                     } catch (Exception e) {
 444                                         logger.error("DesignatedWaiter.run: for PDP {}"
 445                                                 + " Caught Exception attempting to disableFail({})"
 446                                                 + ", message=", pdp.getPdpId(), pdp.getPdpId(), e);
 447                                     }
 448                                 }
 449                             }
 450                         }
 451
 452
 453                     } // end pdps loop
 454
 455                     /*
 456                      * We have checked the four combinations of isDesignated and isCurrent.  Where appropriate,
 457                      * we added the PDPs to the potential list of designated pdps
 458                      *
 459                      * We need to give priority to pdps on the same site that is currently being used
 460                      * First, however, we must sanitize the list of designated to make sure their are
 461                      * only designated members or non-designated members.  There should not be both in
 462                      * the list. Because there are real time delays, it is possible that both types could
 463                      * be on the list.
 464                      */
 465
 466                     listOfDesignated = santizeDesignatedList(listOfDesignated);
 467
 468                     /*
 469                      * We need to figure out the last pdp that was the primary so we can get the last site
 470                      * name and the last session numbers.  We need to create a "dummy" droolspdp since
 471                      * it will be used in later comparisons and cannot be null.
 472                      */
 473
 474                     DroolsPdp mostRecentPrimary = computeMostRecentPrimary(pdps, listOfDesignated);
 475
 476                     if (mostRecentPrimary != null) {
 477                         pdpdLastActive = mostRecentPrimary.getPdpId();
 478                     }
 479
 480
 481                     /*
 482                      * It is possible to get here with more than one pdp designated and providingservice. This normally
 483                      * occurs when there is a race condition with multiple nodes coming up at the same time. If that is
 484                      * the case we must determine which one is the one that should be designated and which one should
 485                      * be demoted.
 486                      *
 487                      * It is possible to have 0, 1, 2 or more but not all, or all designated.
 488                      *   If we have one designated and current, we chose it and are done
 489                      *   If we have 2 or more, but not all, we must determine which one is in the same site as
 490                      *   the previously designated pdp.
 491                      */
 492                     DroolsPdp designatedPdp = computeDesignatedPdp(listOfDesignated, mostRecentPrimary);
 493                     if (designatedPdp != null) {
 494                         pdpdNowActive = designatedPdp.getPdpId();
 495                     }
 496
 497                     if (designatedPdp == null) {
 498                         logger.warn("WARNING: DesignatedWaiter.run: No viable PDP found to be Designated. "
 499                             + "designatedPdp still null.");
 500                         // Just to be sure the parameters are correctly set
 501                         myPdp.setDesignated(false);
 502                         pdpsConnector.setDesignated(myPdp,false);
 503                         isDesignated = false;
 504
 505                         waitTimerLastRunDate = new Date();
 506                         logger.debug("DesignatedWaiter.run (designatedPdp == null) waitTimerLastRunDate = {}",
 507                                         waitTimerLastRunDate);
 508                         myPdp.setUpdatedDate(waitTimerLastRunDate);
 509                         pdpsConnector.update(myPdp);
 510
 511                         return;
 512
 513                     } else if (designatedPdp.getPdpId().equals(myPdp.getPdpId())) {
 514                         logger.debug("DesignatedWaiter.run: designatedPdp is PDP={}", myPdp.getPdpId());
 515                         /*
 516                          * update function expects myPdp.isDesignated to be true.
 517                          */
 518                         try {
 519                             //Keep the order like this.  StateManagement is last since it triggers controller init
 520                             myPdp.setDesignated(true);
 521                             myPdp.setDesignatedDate(new Date());
 522                             pdpsConnector.setDesignated(myPdp, true);
 523                             isDesignated = true;
 524                             String standbyStatus = stateManagementFeature.getStandbyStatus();
 525                             if (!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)) {
 526                                 /*
 527                                  * Only call promote if it is not already in the right state.  Don't worry about
 528                                  * synching the lower level topic endpoint states.  That is done by the
 529                                  * refreshStateAudit.
 530                                  * Note that we need to fetch the session list from 'mostRecentPrimary'
 531                                  * at this point -- soon, 'mostRecentPrimary' will be set to this host.
 532                                  */
 533                                 //this.sessions = mostRecentPrimary.getSessions();
 534                                 stateManagementFeature.promote();
 535                             }
 536                         } catch (Exception e) {
 537                             logger.error("ERROR: DesignatedWaiter.run: Caught Exception attempting to promote PDP={}"
 538                                     + ", message=", myPdp.getPdpId(), e);
 539                             myPdp.setDesignated(false);
 540                             pdpsConnector.setDesignated(myPdp,false);
 541                             isDesignated = false;
 542                             //If you can't promote it, demote it
 543                             try {
 544                                 String standbyStatus = stateManagementFeature.getStandbyStatus();
 545                                 if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 546                                         || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 547                                     /*
 548                                      * Only call demote if it is not already in the right state.  Don't worry about
 549                                      * synching the lower level topic endpoint states.  That is done by the
 550                                      * refreshStateAudit.
 551                                      */
 552                                     stateManagementFeature.demote();
 553                                 }
 554                             } catch (Exception e1) {
 555                                 logger.error("ERROR: DesignatedWaiter.run: Caught StandbyStatusException "
 556                                         + "attempting to promote then demote PDP={}, message=",
 557                                         myPdp.getPdpId(), e1);
 558                             }
 559
 560                         }
 561                         waitTimerLastRunDate = new Date();
 562                         logger.debug("DesignatedWaiter.run (designatedPdp.getPdpId().equals(myPdp.getPdpId())) "
 563                                         + "waitTimerLastRunDate = " + waitTimerLastRunDate);
 564                         myPdp.setUpdatedDate(waitTimerLastRunDate);
 565                         pdpsConnector.update(myPdp);
 566
 567                         return;
 568                     }
 569                     isDesignated = false;
 570
 571                 } // end synchronized
 572                 logger.debug("DesignatedWaiter.run: myPdp: {}; Returning, isDesignated= {}",
 573                                 isDesignated, myPdp.getPdpId());
 574
 575                 Date tmpDate = new Date();
 576                 logger.debug("DesignatedWaiter.run (end of run) waitTimerLastRunDate = {}", tmpDate);
 577
 578                 waitTimerLastRunDate = tmpDate;
 579                 myPdp.setUpdatedDate(waitTimerLastRunDate);
 580                 pdpsConnector.update(myPdp);
 581
 582             } catch (Exception e) {
 583                 logger.error("DesignatedWaiter.run caught an unexpected exception: ", e);
 584             }
 585         } // end run
 586     }
 587
 588     /**
 589      * Sanitize designated list.
 590      *
 591      * @param listOfDesignated list of designated pdps
 592      * @return list of drools pdps
 593      */
 594     public List<DroolsPdp> santizeDesignatedList(List<DroolsPdp> listOfDesignated) {
 595
 596         boolean containsDesignated = false;
 597         boolean containsHotStandby = false;
 598         List<DroolsPdp> listForRemoval = new ArrayList<>();
 599         for (DroolsPdp pdp : listOfDesignated) {
 600             logger.debug("DesignatedWaiter.run sanitizing: pdp = {}"
 601                             + " isDesignated = {}",pdp.getPdpId(), pdp.isDesignated());
 602             if (pdp.isDesignated()) {
 603                 containsDesignated = true;
 604             } else {
 605                 containsHotStandby = true;
 606                 listForRemoval.add(pdp);
 607             }
 608         }
 609         if (containsDesignated && containsHotStandby) {
 610             //remove the hot standby from the list
 611             listOfDesignated.removeAll(listForRemoval);
 612         }
 613         return listOfDesignated;
 614     }
 615
 616     /**
 617      * Compute most recent primary.
 618      *
 619      * @param pdps collection of pdps
 620      * @param listOfDesignated list of designated pdps
 621      * @return drools pdp object
 622      */
 623     public DroolsPdp computeMostRecentPrimary(Collection<DroolsPdp> pdps, List<DroolsPdp> listOfDesignated) {
 624         boolean containsDesignated = false;
 625         for (DroolsPdp pdp : listOfDesignated) {
 626             if (pdp.isDesignated()) {
 627                 containsDesignated = true;
 628             }
 629         }
 630         DroolsPdp mostRecentPrimary = new DroolsPdpImpl(null, true, 1, new Date(0));
 631         mostRecentPrimary.setSite(null);
 632         logger.debug("DesignatedWaiter.run listOfDesignated.size() = {}", listOfDesignated.size());
 633         if (listOfDesignated.size() <= 1) {
 634             logger.debug("DesignatedWainter.run: listOfDesignated.size <=1");
 635             //Only one or none is designated or hot standby.  Choose the latest designated date
 636             for (DroolsPdp pdp : pdps) {
 637                 logger.debug("DesignatedWaiter.run pdp = {}"
 638                                 + " pdp.getDesignatedDate() = {}",
 639                                 pdp.getPdpId(), pdp.getDesignatedDate());
 640                 if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 641                     mostRecentPrimary = pdp;
 642                     logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 643                 }
 644             }
 645         } else if (listOfDesignated.size() == pdps.size()) {
 646             logger.debug("DesignatedWainter.run: listOfDesignated.size = pdps.size() which is {}", pdps.size());
 647             //They are all designated or all hot standby.
 648             mostRecentPrimary = null;
 649             for (DroolsPdp pdp : pdps) {
 650                 if (mostRecentPrimary == null) {
 651                     mostRecentPrimary = pdp;
 652                     continue;
 653                 }
 654                 if (containsDesignated) { //Choose the site of the first designated date
 655                     if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) < 0) {
 656                         mostRecentPrimary = pdp;
 657                         logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 658                     }
 659                 } else { //Choose the site with the latest designated date
 660                     if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 661                         mostRecentPrimary = pdp;
 662                         logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 663                     }
 664                 }
 665             }
 666         } else {
 667             logger.debug("DesignatedWainter.run: Some but not all are designated or hot standby. ");
 668             //Some but not all are designated or hot standby.
 669             if (containsDesignated) {
 670                 logger.debug("DesignatedWainter.run: containsDesignated = {}", containsDesignated);
 671                 /*
 672                  * The list only contains designated.  This is a problem.  It is most likely a race
 673                  * condition that resulted in two thinking they should be designated. Choose the
 674                  * site with the latest designated date for the pdp not included on the designated list.
 675                  * This should be the site that had the last designation before this race condition
 676                  * occurred.
 677                  */
 678                 for (DroolsPdp pdp : pdps) {
 679                     if (listOfDesignated.contains(pdp)) {
 680                         continue; //Don't consider this entry
 681                     }
 682                     if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 683                         mostRecentPrimary = pdp;
 684                         logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 685                     }
 686                 }
 687             } else {
 688                 logger.debug("DesignatedWainter.run: containsDesignated = {}", containsDesignated);
 689                 //The list only contains hot standby. Choose the site of the latest designated date
 690                 for (DroolsPdp pdp : pdps) {
 691                     if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 692                         mostRecentPrimary = pdp;
 693                         logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 694                     }
 695                 }
 696             }
 697         }
 698         return mostRecentPrimary;
 699     }
 700
 701     /**
 702      * Compue designated pdp.
 703      *
 704      * @param listOfDesignated list of designated pdps
 705      * @param mostRecentPrimary most recent primary pdpd
 706      * @return drools pdp object
 707      */
 708     public DroolsPdp computeDesignatedPdp(List<DroolsPdp> listOfDesignated, DroolsPdp mostRecentPrimary) {
 709         DroolsPdp designatedPdp = null;
 710         DroolsPdp lowestPriorityPdp = null;
 711         if (listOfDesignated.size() > 1) {
 712             logger.debug("DesignatedWaiter.run: myPdp: {} listOfDesignated.size(): {}", myPdp.getPdpId(),
 713                             listOfDesignated.size());
 714             DroolsPdp rejectedPdp = null;
 715             DroolsPdp lowestPrioritySameSite = null;
 716             DroolsPdp lowestPriorityDifferentSite = null;
 717             for (DroolsPdp pdp : listOfDesignated) {
 718                 // We need to determine if another PDP is the lowest priority
 719                 if (nullSafeEquals(pdp.getSite(),mostRecentPrimary.getSite())) {
 720                     if (lowestPrioritySameSite == null) {
 721                         if (lowestPriorityDifferentSite != null) {
 722                             rejectedPdp = lowestPriorityDifferentSite;
 723                         }
 724                         lowestPrioritySameSite = pdp;
 725                     } else {
 726                         if (pdp.getPdpId().equals((lowestPrioritySameSite.getPdpId()))) {
 727                             continue;//nothing to compare
 728                         }
 729                         if (pdp.comparePriority(lowestPrioritySameSite) < 0) {
 730                             logger.debug("\nDesignatedWaiter.run: myPdp {}  listOfDesignated pdp ID: {}"
 731                                             + " has lower priority than pdp ID: {}",myPdp.getPdpId(), pdp.getPdpId(),
 732                                             lowestPrioritySameSite.getPdpId());
 733                             //we need to reject lowestPrioritySameSite
 734                             rejectedPdp = lowestPrioritySameSite;
 735                             lowestPrioritySameSite = pdp;
 736                         } else {
 737                             //we need to reject pdp and keep lowestPrioritySameSite
 738                             logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {} "
 739                                             + " has higher priority than pdp ID: {}", myPdp.getPdpId(),pdp.getPdpId(),
 740                                             lowestPrioritySameSite.getPdpId());
 741                             rejectedPdp = pdp;
 742                         }
 743                     }
 744                 } else {
 745                     if (lowestPrioritySameSite != null) {
 746                         //if we already have a candidate for same site, we don't want to bother with different sites
 747                         rejectedPdp = pdp;
 748                     } else {
 749                         if (lowestPriorityDifferentSite == null) {
 750                             lowestPriorityDifferentSite = pdp;
 751                             continue;
 752                         }
 753                         if (pdp.getPdpId().equals((lowestPriorityDifferentSite.getPdpId()))) {
 754                             continue;//nothing to compare
 755                         }
 756                         if (pdp.comparePriority(lowestPriorityDifferentSite) < 0) {
 757                             logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 758                                             + " has lower priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 759                                             lowestPriorityDifferentSite.getPdpId());
 760                             //we need to reject lowestPriorityDifferentSite
 761                             rejectedPdp = lowestPriorityDifferentSite;
 762                             lowestPriorityDifferentSite = pdp;
 763                         } else {
 764                             //we need to reject pdp and keep lowestPriorityDifferentSite
 765                             logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 766                                             + " has higher priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 767                                             lowestPriorityDifferentSite.getPdpId());
 768                             rejectedPdp = pdp;
 769                         }
 770                     }
 771                 }
 772                 // If the rejectedPdp is myPdp, we need to stand it down and demote it.  Each pdp is responsible
 773                 // for demoting itself
 774                 if (rejectedPdp != null && nullSafeEquals(rejectedPdp.getPdpId(),myPdp.getPdpId())) {
 775                     logger.debug("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated myPdp ID: {}"
 776                                     + " is NOT the lowest priority.  Executing stateManagement.demote()\n\n",
 777                                     myPdp.getPdpId(),
 778                                     myPdp.getPdpId());
 779                     // We found that myPdp is on the listOfDesignated and it is not the lowest priority
 780                     // So, we must demote it
 781                     try {
 782                         //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 783                         myPdp.setDesignated(false);
 784                         pdpsConnector.setDesignated(myPdp, false);
 785                         isDesignated = false;
 786                         String standbyStatus = stateManagementFeature.getStandbyStatus();
 787                         if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 788                                 || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 789                             /*
 790                              * Only call demote if it is not already in the right state.  Don't worry about
 791                              * synching the lower level topic endpoint states.  That is done by the
 792                              * refreshStateAudit.
 793                              */
 794                             stateManagementFeature.demote();
 795                         }
 796                     } catch (Exception e) {
 797                         myPdp.setDesignated(false);
 798                         pdpsConnector.setDesignated(myPdp, false);
 799                         isDesignated = false;
 800                         logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 801                                 + "demote myPdp {} myPdp.getPdpId(), message= {}", myPdp.getPdpId(),
 802                                 e);
 803                     }
 804                 }
 805             } //end: for(DroolsPdp pdp : listOfDesignated)
 806             if (lowestPrioritySameSite != null) {
 807                 lowestPriorityPdp = lowestPrioritySameSite;
 808             } else {
 809                 lowestPriorityPdp = lowestPriorityDifferentSite;
 810             }
 811             //now we have a valid value for lowestPriorityPdp
 812             logger.debug("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 813                             + "found the LOWEST priority pdp ID: {} "
 814                             + " It is now the designatedPpd from the perspective of myPdp ID: {} \n\n",
 815                             myPdp.getPdpId(), lowestPriorityPdp.getPdpId(), myPdp);
 816             designatedPdp = lowestPriorityPdp;
 817
 818         } else if (listOfDesignated.isEmpty()) {
 819             logger.debug("\nDesignatedWaiter.run: myPdp: {} listOfDesignated is: EMPTY.", myPdp.getPdpId());
 820             designatedPdp = null;
 821         } else { //only one in listOfDesignated
 822             logger.debug("\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 823                             + "has ONE entry. PDP ID: {}", myPdp.getPdpId(), listOfDesignated.get(0).getPdpId());
 824             designatedPdp = listOfDesignated.get(0);
 825         }
 826         return designatedPdp;
 827
 828     }
 829
 830     private class TimerUpdateClass extends TimerTask {
 831
 832         @Override
 833         public void run() {
 834             try {
 835                 logger.debug("TimerUpdateClass.run: entry");
 836                 checkWaitTimer();
 837             } catch (Exception e) {
 838                 logger.error("TimerUpdateClass.run caught an unexpected exception: ", e);
 839             }
 840             logger.debug("TimerUpdateClass.run.exit");
 841         }
 842     }
 843
 844     @Override
 845     public void checkThreadStatus() {
 846         checkWaitTimer();
 847     }
 848
 849     private void checkWaitTimer() {
 850         synchronized (checkWaitTimerLock) {
 851             try {
 852                 logger.debug("checkWaitTimer: entry");
 853                 Date now = new Date();
 854                 long nowMs = now.getTime();
 855                 long waitTimerMs = waitTimerLastRunDate.getTime();
 856
 857                 //give it 10 times leeway
 858                 if ((nowMs - waitTimerMs)  > 10 * pdpUpdateInterval) {
 859                     if (allSeemsWell == null || allSeemsWell) {
 860                         allSeemsWell = false;
 861                         logger.debug("checkWaitTimer: calling allSeemsWell with ALLNOTWELL param");
 862                         stateManagementFeature.allSeemsWell(this.getClass().getName(),
 863                                 StateManagementFeatureApiConstants.ALLNOTWELL_STATE,
 864                                 "DesignationWaiter/ElectionHandler has STALLED");
 865                     }
 866                     logger.error("checkWaitTimer: nowMs - waitTimerMs = {}"
 867                             + ", exceeds 10* pdpUpdateInterval = {}"
 868                             + " DesignationWaiter is STALLED!", (nowMs - waitTimerMs), (10 * pdpUpdateInterval));
 869                 } else if (allSeemsWell == null || !allSeemsWell) {
 870                     allSeemsWell = true;
 871                     stateManagementFeature.allSeemsWell(this.getClass().getName(),
 872                             StateManagementFeatureApiConstants.ALLSEEMSWELL_STATE,
 873                             "DesignationWaiter/ElectionHandler has RESUMED");
 874                     logger.info("DesignationWaiter/ElectionHandler has RESUMED");
 875                 }
 876                 logger.debug("checkWaitTimer: exit");
 877             } catch (Exception e) {
 878                 logger.error("checkWaitTimer: caught unexpected exception: ", e);
 879             }
 880         }
 881     }
 882
 883     private long getDWaiterStartMs() {
 884         Date now = new Date();
 885
 886         // Retrieve the ms since the epoch
 887         long nowMs = now.getTime();
 888
 889         // Time since the end of the last pdpUpdateInterval multiple
 890         long nowModMs = nowMs % pdpUpdateInterval;
 891
 892         // Time to the start of the next pdpUpdateInterval multiple
 893         long startMs = 2 * pdpUpdateInterval - nowModMs;
 894
 895         // Give the start time a minimum of a 5 second cushion
 896         if (startMs < 5000) {
 897             // Start at the beginning  of following interval
 898             startMs = pdpUpdateInterval + startMs;
 899         }
 900         return startMs;
 901     }
 902
 903     private boolean nullSafeEquals(Object one, Object two) {
 904         if (one == null && two == null) {
 905             return true;
 906         }
 907         if (one != null && two != null) {
 908             return one.equals(two);
 909         }
 910         return false;
 911     }
 912
 913     public String getPdpdNowActive() {
 914         return pdpdNowActive;
 915     }
 916
 917     public String getPdpdLastActive() {
 918         return pdpdLastActive;
 919     }
 920 }