gerrit.onap Code Review - policy/drools-pdp.git/blob

   1 /*
   2  * ============LICENSE_START=======================================================
   3  * feature-active-standby-management
   4  * ================================================================================
   5  * Copyright (C) 2017-2019 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.policy.drools.activestandby;
  22
  23 import java.util.ArrayList;
  24 import java.util.Collection;
  25 import java.util.Date;
  26 import java.util.List;
  27 import java.util.Timer;
  28 import java.util.TimerTask;
  29
  30 import org.onap.policy.common.im.StateManagement;
  31 import org.onap.policy.drools.statemanagement.StateManagementFeatureApi;
  32 import org.onap.policy.drools.statemanagement.StateManagementFeatureApiConstants;
  33 import org.slf4j.Logger;
  34 import org.slf4j.LoggerFactory;
  35
  36 public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
  37     private static final String RUN_PRIMARY_MSG = "DesignatedWaiter.run mostRecentPrimary = {}";
  38
  39     // get an instance of logger
  40     private static final Logger  logger = LoggerFactory.getLogger(DroolsPdpsElectionHandler.class);
  41     private DroolsPdpsConnector pdpsConnector;
  42     private Object checkWaitTimerLock = new Object();
  43     private Object designationWaiterLock = new Object();
  44
  45     /*
  46      * Must be static, so it can be referenced by JpaDroolsPdpsConnector,
  47      * without requiring a reference to the election handler instantiation.
  48      */
  49     private static DroolsPdp myPdp;
  50
  51     private DesignationWaiter designationWaiter;
  52     private Timer updateWorker;
  53     private Timer waitTimer;
  54     private Date waitTimerLastRunDate;
  55
  56     // The interval between checks of the DesignationWaiter to be sure it is running.
  57     private int pdpCheckInterval;
  58
  59     // The interval between runs of the DesignationWaiter
  60     private int pdpUpdateInterval;
  61
  62     private volatile boolean isDesignated;
  63
  64     private String pdpdNowActive;
  65     private String pdpdLastActive;
  66
  67     /*
  68      * Start allSeemsWell with a value of null so that, on the first run
  69      * of the checkWaitTimer it will set the value in IntegrityMonitor
  70      * regardless of whether it needs to be set to true or false.
  71      */
  72     private Boolean allSeemsWell = null;
  73
  74     private StateManagementFeatureApi stateManagementFeature;
  75
  76     private static boolean isUnitTesting = false;
  77     private static boolean isStalled = false;
  78
  79     /**
  80      * Constructor.
  81      *
  82      * @param pdps connectors
  83      * @param myPdp pdp
  84      */
  85     public DroolsPdpsElectionHandler(DroolsPdpsConnector pdps, DroolsPdp myPdp) {
  86         if (pdps == null) {
  87             logger.error("DroolsPdpsElectinHandler(): pdpsConnector==null");
  88             throw new IllegalArgumentException("DroolsPdpsElectinHandler(): pdpsConnector==null");
  89         }
  90         if (myPdp == null) {
  91             logger.error("DroolsPdpsElectinHandler(): droolsPdp==null");
  92             throw new IllegalArgumentException("DroolsPdpsElectinHandler(): DroolsPdp==null");
  93         }
  94
  95         pdpdNowActive = null;
  96         pdpdLastActive = null;
  97         this.pdpsConnector = pdps;
  98         DroolsPdpsElectionHandler.myPdp = myPdp;
  99         this.isDesignated = false;
 100         pdpCheckInterval = 3000;
 101         try {
 102             pdpCheckInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(
 103                     ActiveStandbyProperties.PDP_CHECK_INVERVAL));
 104         } catch (Exception e) {
 105             logger.error("Could not get pdpCheckInterval property. Using default {}",pdpCheckInterval, e);
 106         }
 107         pdpUpdateInterval = 2000;
 108         try {
 109             pdpUpdateInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(
 110                     ActiveStandbyProperties.PDP_UPDATE_INTERVAL));
 111         } catch (Exception e) {
 112             logger.error("Could not get pdpUpdateInterval property. Using default {} ", pdpUpdateInterval, e);
 113         }
 114
 115         Date now = new Date();
 116
 117         // Retrieve the ms since the epoch
 118         final long nowMs = now.getTime();
 119
 120         // Create the timer which will update the updateDate in DroolsPdpEntity table.
 121         // This is the heartbeat
 122         updateWorker = new Timer();
 123
 124         // Schedule the TimerUpdateClass to run at 100 ms and run at pdpCheckInterval ms thereafter
 125         // NOTE: The first run of the TimerUpdateClass results in myPdp being added to the
 126         // drools droolsPdpEntity table.
 127         updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
 128
 129         // Create the timer which will run the election algorithm
 130         waitTimer = new Timer();
 131
 132         // Schedule it to start in startMs ms
 133         // (so it will run after the updateWorker and run at pdpUpdateInterval ms thereafter
 134         long startMs = getDWaiterStartMs();
 135         designationWaiter = new DesignationWaiter();
 136         waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
 137         waitTimerLastRunDate = new Date(nowMs + startMs);
 138
 139         //Get the StateManagementFeature instance
 140
 141         for (StateManagementFeatureApi feature : StateManagementFeatureApiConstants.getImpl().getList()) {
 142             if (feature.getResourceName().equals(myPdp.getPdpId())) {
 143                 logger.debug("DroolsPdpsElectionHandler: Found StateManagementFeature"
 144                                 + " with resourceName: {}", myPdp.getPdpId());
 145                 stateManagementFeature = feature;
 146                 break;
 147             }
 148         }
 149         if (stateManagementFeature == null) {
 150             logger.error("DroolsPdpsElectionHandler failed to initialize.  "
 151                     + "Unable to get instance of StateManagementFeatureApi "
 152                     + "with resourceID: {}", myPdp.getPdpId());
 153         }
 154     }
 155
 156     public static void setIsUnitTesting(boolean val) {
 157         isUnitTesting = val;
 158     }
 159
 160     public static void setIsStalled(boolean val) {
 161         isStalled = val;
 162     }
 163
 164     /**
 165      * When the JpaDroolsPdpsConnector.standDown() method is invoked, it needs
 166      * access to myPdp, so it can keep its designation status in sync with the
 167      * DB.
 168      *
 169      * @param designated is designated value
 170      */
 171     public static void setMyPdpDesignated(boolean designated) {
 172         logger.debug("setMyPdpDesignated: designated= {}", designated);
 173         myPdp.setDesignated(designated);
 174     }
 175
 176     private class DesignationWaiter extends TimerTask {
 177         // get an instance of logger
 178         private final Logger  logger = LoggerFactory.getLogger(DesignationWaiter.class);
 179
 180         @Override
 181         public void run() {
 182             try {
 183                 logger.debug("DesignatedWaiter.run: Entering");
 184
 185                 //This is for testing the checkWaitTimer
 186                 if (isUnitTesting && isStalled) {
 187                     logger.debug("DesignatedWaiter.run: isUnitTesting = {} isStalled = {}",
 188                                     isUnitTesting, isStalled);
 189                     return;
 190                 }
 191
 192                 synchronized (designationWaiterLock) {
 193
 194                     logger.debug("DesignatedWaiter.run: Entering synchronized block");
 195
 196                     //It is possible that multiple PDPs are designated lead.  So, we will make a list of all designated
 197                     //PDPs and then decide which one really should be designated at the end.
 198                     List<DroolsPdp> listOfDesignated = new ArrayList<>();
 199
 200                     Collection<DroolsPdp> pdps = pdpsConnector.getDroolsPdps();
 201
 202                     logger.debug("DesignatedWaiter.run: pdps.size= {}", pdps.size());
 203
 204                     //This is only true if all designated PDPs have failed
 205                     allPdpsFailed(pdps, listOfDesignated);
 206
 207                     /*
 208                      * We have checked the four combinations of isDesignated and isCurrent.  Where appropriate,
 209                      * we added the PDPs to the potential list of designated pdps
 210                      *
 211                      * We need to give priority to pdps on the same site that is currently being used
 212                      * First, however, we must sanitize the list of designated to make sure their are
 213                      * only designated members or non-designated members.  There should not be both in
 214                      * the list. Because there are real time delays, it is possible that both types could
 215                      * be on the list.
 216                      */
 217
 218                     listOfDesignated = santizeDesignatedList(listOfDesignated);
 219
 220                     /*
 221                      * We need to figure out the last pdp that was the primary so we can get the last site
 222                      * name and the last session numbers.  We need to create a "dummy" droolspdp since
 223                      * it will be used in later comparisons and cannot be null.
 224                      */
 225
 226                     DroolsPdp mostRecentPrimary = computeMostRecentPrimary(pdps, listOfDesignated);
 227
 228                     if (mostRecentPrimary != null) {
 229                         pdpdLastActive = mostRecentPrimary.getPdpId();
 230                     }
 231
 232
 233                     /*
 234                      * It is possible to get here with more than one pdp designated and providing service. This normally
 235                      * occurs when there is a race condition with multiple nodes coming up at the same time. If that is
 236                      * the case we must determine which one is the one that should be designated and which one should
 237                      * be demoted.
 238                      *
 239                      * It is possible to have 0, 1, 2 or more but not all, or all designated.
 240                      *   If we have one designated and current, we chose it and are done
 241                      *   If we have 2 or more, but not all, we must determine which one is in the same site as
 242                      *   the previously designated pdp.
 243                      */
 244                     DroolsPdp designatedPdp = computeDesignatedPdp(listOfDesignated, mostRecentPrimary);
 245
 246                     if (designatedPdp == null) {
 247                         logger.warn("WARNING: DesignatedWaiter.run: No viable PDP found to be Designated. "
 248                             + "designatedPdp still null.");
 249                         designateNoPdp();
 250                         return;
 251                     }
 252
 253                     pdpdNowActive = designatedPdp.getPdpId();
 254
 255                     if (pdpdNowActive.equals(myPdp.getPdpId())) {
 256                         logger.debug("DesignatedWaiter.run: designatedPdp is PDP={}", myPdp.getPdpId());
 257                         designateMyPdp();
 258                         return;
 259                     }
 260
 261                     isDesignated = false;
 262
 263                 } // end synchronized
 264                 logger.debug("DesignatedWaiter.run: myPdp: {}; Returning, isDesignated= {}",
 265                                 isDesignated, myPdp.getPdpId());
 266
 267                 Date tmpDate = new Date();
 268                 logger.debug("DesignatedWaiter.run (end of run) waitTimerLastRunDate = {}", tmpDate);
 269
 270                 waitTimerLastRunDate = tmpDate;
 271                 myPdp.setUpdatedDate(waitTimerLastRunDate);
 272                 pdpsConnector.update(myPdp);
 273
 274             } catch (Exception e) {
 275                 logger.error("DesignatedWaiter.run caught an unexpected exception: ", e);
 276             }
 277         } // end run
 278
 279         private void allPdpsFailed(Collection<DroolsPdp> pdps, List<DroolsPdp> listOfDesignated) {
 280             boolean designatedPdpHasFailed = pdpsConnector.hasDesignatedPdpFailed(pdps);
 281             logger.debug("DesignatedWaiter.run: designatedPdpHasFailed= {}", designatedPdpHasFailed);
 282             for (DroolsPdp pdp : pdps) {
 283                 logger.debug("DesignatedWaiter.run: evaluating pdp ID: {}", pdp.getPdpId());
 284
 285                 /*
 286                  * Note: side effect of isPdpCurrent is that any stale but
 287                  * designated PDPs will be marked as un-designated.
 288                  */
 289                 boolean isCurrent = pdpsConnector.isPdpCurrent(pdp);
 290
 291                 /*
 292                  * We can't use stateManagement.getStandbyStatus() here, because
 293                  * we need the standbyStatus, not for this PDP, but for the PDP
 294                  * being processed by this loop iteration.
 295                  */
 296                 String standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 297                 if (standbyStatus == null) {
 298                     // Treat this case as a cold standby -- if we
 299                     // abort here, no sessions will be created in a
 300                     // single-node test environment.
 301                     standbyStatus = StateManagement.COLD_STANDBY;
 302                 }
 303                 logger.debug("DesignatedWaiter.run: PDP= {},  isCurrent= {}", pdp.getPdpId(), isCurrent);
 304
 305                 adjustPdp(pdp, isCurrent, designatedPdpHasFailed, standbyStatus, listOfDesignated);
 306
 307
 308             } // end pdps loop
 309         }
 310
 311         private void adjustPdp(DroolsPdp pdp, boolean isCurrent, boolean designatedPdpHasFailed, String standbyStatus,
 312                         List<DroolsPdp> listOfDesignated) {
 313             /*
 314              * There are 4 combinations of isDesignated and isCurrent.  We will examine each one in-turn
 315              * and evaluate the each pdp in the list of pdps against each combination.
 316              */
 317             if (pdp.isDesignated()) {
 318                 /*
 319                  * This is the first combination of isDesignated and isCurrent
 320                  */
 321                 if (isCurrent) {
 322                     pdpDesignatedCurrent(pdp, standbyStatus, listOfDesignated);
 323
 324                 /*
 325                  * The second combination of isDesignated and isCurrent
 326                  *
 327                  * PDP is designated but not current; it has failed.
 328                  * So we stand it down (it doesn't matter what
 329                  * its standbyStatus is). None of these go on the list.
 330                  */
 331                 } else {
 332                     logger.debug("INFO: DesignatedWaiter.run: PDP= {} is currently "
 333                                     + "designated but is not current; "
 334                                     + "it has failed.  Standing down.  standbyStatus= {}",
 335                                     pdp.getPdpId(), standbyStatus);
 336                     pdpDesignatedNotCurrent(pdp);
 337                 }
 338
 339             } else {
 340                 // NOT designated
 341
 342
 343                 /*
 344                  * The third combination of isDesignated and isCurrent
 345                  * /*
 346                  * If a PDP is not currently designated but is providing service
 347                  * (erroneous, but recoverable) or hot standby
 348                  * we can add it to the list of possible designated if all the designated have failed
 349                  */
 350                 if (isCurrent) {
 351                     pdpNotDesignatedCurrent(pdp, designatedPdpHasFailed, standbyStatus,
 352                                     listOfDesignated);
 353
 354                 /*
 355                  * The fourth combination of isDesignated and isCurrent
 356                  *
 357                  * We are not going to put any of these on the list since it appears they have failed.
 358                  *
 359                  */
 360                 } else {
 361                     logger.debug("INFO: DesignatedWaiter.run: PDP= {} "
 362                                     + "designated= {}, current= {}, "
 363                                     + "designatedPdpHasFailed= {}, "
 364                                     + "standbyStatus= {}",pdp.getPdpId(),
 365                                     pdp.isDesignated(), false, designatedPdpHasFailed, standbyStatus);
 366                     pdpNotDesignatedNotCurrent(pdp, standbyStatus);
 367                 }
 368             }
 369         }
 370
 371         private void pdpDesignatedCurrent(DroolsPdp pdp, String standbyStatus, List<DroolsPdp> listOfDesignated) {
 372             //It is current, but it could have a standbystatus=coldstandby / hotstandby
 373             //If so, we need to stand it down and demote it
 374             if (!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)) {
 375                 if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 376                     logger.debug("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 377                                     + "butstandbystatus is not providingservice. "
 378                                     + " Executing stateManagement.demote()" + "\n\n", myPdp.getPdpId());
 379                     // So, we must demote it
 380                     try {
 381                         demoteMyPdp(pdp, standbyStatus);
 382                     } catch (Exception e) {
 383                         logger.error("DesignatedWaiter.run: myPdp: {} "
 384                                 + "Caught Exception attempting to demote myPdp,"
 385                                 + "message= {}", myPdp.getPdpId(), e);
 386                     }
 387                 } else {
 388                     // Don't demote a remote PDP that is current.  It should catch itself
 389                     logger.debug("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 390                                     + "but standbystatus is not providingservice. "
 391                                     + " Cannot execute stateManagement.demote() "
 392                                     + "since it it is not myPdp\n\n",
 393                                     myPdp.getPdpId());
 394                 }
 395
 396             } else {
 397                 // If we get here, it is ok to be on the list
 398                 logger.debug("DesignatedWaiter.run: PDP= {} is designated, "
 399                                 + "current and {} Noting PDP as "
 400                                 + "designated, standbyStatus= {}",
 401                                 pdp.getPdpId(), standbyStatus, standbyStatus);
 402                 listOfDesignated.add(pdp);
 403             }
 404         }
 405
 406         private void demoteMyPdp(DroolsPdp pdp, String standbyStatus) throws Exception {
 407             //Keep the order like this.  StateManagement is last since it
 408             //triggers controller shutdown
 409             //This will change isDesignated and it can enter another if(combination) below
 410             pdpsConnector.standDownPdp(pdp.getPdpId());
 411             myPdp.setDesignated(false);
 412             isDesignated = false;
 413             if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 414                     || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 415                 /*
 416                  * Only demote it if it appears it has not already been demoted. Don't worry
 417                  * about synching with the topic endpoint states.  That is done by the
 418                  * refreshStateAudit
 419                  */
 420                 stateManagementFeature.demote();
 421             }
 422         }
 423
 424         private void pdpDesignatedNotCurrent(DroolsPdp pdp) {
 425             /*
 426              * Changes designated to 0 but it is still potentially providing service
 427              * Will affect isDesignated, so, it can enter an if(combination) below
 428              */
 429             pdpsConnector.standDownPdp(pdp.getPdpId());
 430
 431             //need to change standbystatus to coldstandby
 432             if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 433                 logger.debug("\n\nDesignatedWaiter.run: myPdp {} is not Current. "
 434                                 + " Executing stateManagement.disableFailed()\n\n", myPdp.getPdpId());
 435                 // We found that myPdp is designated but not current
 436                 // So, we must cause it to disableFail
 437                 try {
 438                     myPdp.setDesignated(false);
 439                     pdpsConnector.setDesignated(myPdp, false);
 440                     isDesignated = false;
 441                     stateManagementFeature.disableFailed();
 442                 } catch (Exception e) {
 443                     logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception "
 444                             + "attempting to disableFail myPdp {}, message= {}",
 445                             myPdp.getPdpId(), myPdp.getPdpId(), e);
 446                 }
 447             } else { //it is a remote PDP that is failed
 448                 logger.debug("\n\nDesignatedWaiter.run: PDP {} is not Current. "
 449                                 + " Executing stateManagement.disableFailed(otherResourceName)\n\n",
 450                                 pdp.getPdpId() );
 451                 // We found a PDP is designated but not current
 452                 // We already called standdown(pdp) which will change designated to false
 453                 // Now we need to disableFail it to get its states in synch.  The standbyStatus
 454                 // should equal coldstandby
 455                 try {
 456                     stateManagementFeature.disableFailed(pdp.getPdpId());
 457                 } catch (Exception e) {
 458                     logger.error("DesignatedWaiter.run: for PDP {}  Caught Exception attempting to "
 459                             + "disableFail({}), message= {}",
 460                             pdp.getPdpId(), pdp.getPdpId(), e);
 461                 }
 462
 463             }
 464         }
 465
 466         private void pdpNotDesignatedCurrent(DroolsPdp pdp, boolean designatedPdpHasFailed, String standbyStatus,
 467                         List<DroolsPdp> listOfDesignated) {
 468             if (!(StateManagement.HOT_STANDBY.equals(standbyStatus)
 469                     || StateManagement.COLD_STANDBY.equals(standbyStatus))) {
 470                 logger.debug("\n\nDesignatedWaiter.run: PDP {}"
 471                                 + " is NOT designated but IS current and"
 472                                 + " has a standbystatus= {}", pdp.getPdpId(), standbyStatus);
 473                 // Since it is current, we assume it can adjust its own state.
 474                 // We will demote if it is myPdp
 475                 if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 476                     //demote it
 477                     logger.debug("DesignatedWaiter.run: PDP {} going to "
 478                                     + "setDesignated = false and calling stateManagement.demote",
 479                                     pdp.getPdpId());
 480                     try {
 481                         //Keep the order like this.
 482                         //StateManagement is last since it triggers controller shutdown
 483                         pdpsConnector.setDesignated(myPdp, false);
 484                         myPdp.setDesignated(false);
 485                         isDesignated = false;
 486                         //This is definitely not a redundant call.
 487                         //It is attempting to correct a problem
 488                         stateManagementFeature.demote();
 489                         //recheck the standbystatus
 490                         standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 491                     } catch (Exception e) {
 492                         logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception "
 493                                 + "attempting to demote myPdp {}, message = {}",  myPdp.getPdpId(),
 494                                 myPdp.getPdpId(), e);
 495                     }
 496
 497                 }
 498             }
 499             if (StateManagement.HOT_STANDBY.equals(standbyStatus) && designatedPdpHasFailed) {
 500                 //add it to the list
 501                 logger.debug("INFO: DesignatedWaiter.run: PDP= {}"
 502                                 + " is not designated but is {} and designated PDP "
 503                                 + "has failed.  standbyStatus= {}", pdp.getPdpId(),
 504                                 standbyStatus, standbyStatus);
 505                 listOfDesignated.add(pdp);
 506             }
 507         }
 508
 509         private void pdpNotDesignatedNotCurrent(DroolsPdp pdp, String standbyStatus) {
 510             if (StateManagement.COLD_STANDBY.equals(standbyStatus)) {
 511                 return;
 512             }
 513
 514             //stand it down
 515             //disableFail it
 516             pdpsConnector.standDownPdp(pdp.getPdpId());
 517             if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 518                 /*
 519                  * I don't actually know how this condition could
 520                  * happen, but if it did, we would want to declare it
 521                  * failed.
 522                  */
 523                 logger.debug("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 524                                 + " Executing stateManagement.disableFailed()\n\n",
 525                                 myPdp.getPdpId());
 526                 // So, we must disableFail it
 527                 try {
 528                     //Keep the order like this.
 529                     //StateManagement is last since it triggers controller shutdown
 530                     pdpsConnector.setDesignated(myPdp, false);
 531                     myPdp.setDesignated(false);
 532                     isDesignated = false;
 533                     stateManagementFeature.disableFailed();
 534                 } catch (Exception e) {
 535                     logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 536                             + "disableFail myPdp {}, message= {}",
 537                             myPdp.getPdpId(), myPdp.getPdpId(), e);
 538                 }
 539             } else { //it is remote
 540                 logger.debug("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 541                                 + " Executing stateManagement.disableFailed({})\n\n",
 542                                 myPdp.getPdpId(), pdp.getPdpId());
 543                 // We already called standdown(pdp) which will change designated to false
 544                 // Now we need to disableFail it to get its states in sync.
 545                 // StandbyStatus = coldstandby
 546                 try {
 547                     stateManagementFeature.disableFailed(pdp.getPdpId());
 548                 } catch (Exception e) {
 549                     logger.error("DesignatedWaiter.run: for PDP {}"
 550                             + " Caught Exception attempting to disableFail({})"
 551                             + ", message=", pdp.getPdpId(), pdp.getPdpId(), e);
 552                 }
 553             }
 554         }
 555
 556         private void designateNoPdp() {
 557             // Just to be sure the parameters are correctly set
 558             myPdp.setDesignated(false);
 559             pdpsConnector.setDesignated(myPdp,false);
 560             isDesignated = false;
 561
 562             waitTimerLastRunDate = new Date();
 563             logger.debug("DesignatedWaiter.run (designatedPdp == null) waitTimerLastRunDate = {}",
 564                             waitTimerLastRunDate);
 565             myPdp.setUpdatedDate(waitTimerLastRunDate);
 566             pdpsConnector.update(myPdp);
 567         }
 568
 569         private void designateMyPdp() {
 570             /*
 571              * update function expects myPdp.isDesignated to be true.
 572              */
 573             try {
 574                 //Keep the order like this.  StateManagement is last since it triggers controller init
 575                 myPdp.setDesignated(true);
 576                 myPdp.setDesignatedDate(new Date());
 577                 pdpsConnector.setDesignated(myPdp, true);
 578                 isDesignated = true;
 579                 String standbyStatus = stateManagementFeature.getStandbyStatus();
 580                 if (!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)) {
 581                     /*
 582                      * Only call promote if it is not already in the right state.  Don't worry about
 583                      * synching the lower level topic endpoint states.  That is done by the
 584                      * refreshStateAudit.
 585                      * Note that we need to fetch the session list from 'mostRecentPrimary'
 586                      * at this point -- soon, 'mostRecentPrimary' will be set to this host.
 587                      */
 588                     //this.sessions = mostRecentPrimary.getSessions();
 589                     stateManagementFeature.promote();
 590                 }
 591             } catch (Exception e) {
 592                 logger.error("ERROR: DesignatedWaiter.run: Caught Exception attempting to promote PDP={}"
 593                         + ", message=", myPdp.getPdpId(), e);
 594                 myPdp.setDesignated(false);
 595                 pdpsConnector.setDesignated(myPdp,false);
 596                 isDesignated = false;
 597                 //If you can't promote it, demote it
 598                 try {
 599                     String standbyStatus = stateManagementFeature.getStandbyStatus();
 600                     if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 601                             || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 602                         /*
 603                          * Only call demote if it is not already in the right state.  Don't worry about
 604                          * synching the lower level topic endpoint states.  That is done by the
 605                          * refreshStateAudit.
 606                          */
 607                         stateManagementFeature.demote();
 608                     }
 609                 } catch (Exception e1) {
 610                     logger.error("ERROR: DesignatedWaiter.run: Caught StandbyStatusException "
 611                             + "attempting to promote then demote PDP={}, message=",
 612                             myPdp.getPdpId(), e1);
 613                 }
 614
 615             }
 616             waitTimerLastRunDate = new Date();
 617             logger.debug("DesignatedWaiter.run (designatedPdp.getPdpId().equals(myPdp.getPdpId())) "
 618                             + "waitTimerLastRunDate = " + waitTimerLastRunDate);
 619             myPdp.setUpdatedDate(waitTimerLastRunDate);
 620             pdpsConnector.update(myPdp);
 621         }
 622     }
 623
 624     /**
 625      * Sanitize designated list.
 626      *
 627      * @param listOfDesignated list of designated pdps
 628      * @return list of drools pdps
 629      */
 630     public List<DroolsPdp> santizeDesignatedList(List<DroolsPdp> listOfDesignated) {
 631
 632         boolean containsDesignated = false;
 633         boolean containsHotStandby = false;
 634         List<DroolsPdp> listForRemoval = new ArrayList<>();
 635         for (DroolsPdp pdp : listOfDesignated) {
 636             logger.debug("DesignatedWaiter.run sanitizing: pdp = {}"
 637                             + " isDesignated = {}",pdp.getPdpId(), pdp.isDesignated());
 638             if (pdp.isDesignated()) {
 639                 containsDesignated = true;
 640             } else {
 641                 containsHotStandby = true;
 642                 listForRemoval.add(pdp);
 643             }
 644         }
 645         if (containsDesignated && containsHotStandby) {
 646             //remove the hot standby from the list
 647             listOfDesignated.removeAll(listForRemoval);
 648         }
 649         return listOfDesignated;
 650     }
 651
 652     /**
 653      * Compute most recent primary.
 654      *
 655      * @param pdps collection of pdps
 656      * @param listOfDesignated list of designated pdps
 657      * @return drools pdp object
 658      */
 659     public DroolsPdp computeMostRecentPrimary(Collection<DroolsPdp> pdps, List<DroolsPdp> listOfDesignated) {
 660         boolean containsDesignated = listOfDesignated.stream().anyMatch(DroolsPdp::isDesignated);
 661
 662         DroolsPdp mostRecentPrimary = new DroolsPdpImpl(null, true, 1, new Date(0));
 663         mostRecentPrimary.setSite(null);
 664         logger.debug("DesignatedWaiter.run listOfDesignated.size() = {}", listOfDesignated.size());
 665
 666         if (listOfDesignated.size() <= 1) {
 667             logger.debug("DesignatedWainter.run: listOfDesignated.size <=1");
 668             //Only one or none is designated or hot standby.  Choose the latest designated date
 669             mostRecentPrimary = getLatestDesignated(pdps, mostRecentPrimary);
 670
 671         } else if (listOfDesignated.size() == pdps.size()) {
 672             logger.debug("DesignatedWainter.run: listOfDesignated.size = pdps.size() which is {}", pdps.size());
 673             //They are all designated or all hot standby.
 674             mostRecentPrimary = getBestDesignated(pdps, containsDesignated);
 675
 676         } else {
 677             logger.debug("DesignatedWainter.run: Some but not all are designated or hot standby. ");
 678             logger.debug("DesignatedWainter.run: containsDesignated = {}", containsDesignated);
 679             //Some but not all are designated or hot standby.
 680             if (containsDesignated) {
 681                 /*
 682                  * The list only contains designated.  This is a problem.  It is most likely a race
 683                  * condition that resulted in two thinking they should be designated. Choose the
 684                  * site with the latest designated date for the pdp not included on the designated list.
 685                  * This should be the site that had the last designation before this race condition
 686                  * occurred.
 687                  */
 688                 mostRecentPrimary = getLatestUndesignated(pdps, mostRecentPrimary, listOfDesignated);
 689
 690             } else {
 691                 //The list only contains hot standby. Choose the site of the latest designated date
 692                 mostRecentPrimary = getLatestDesignated(pdps, mostRecentPrimary);
 693             }
 694         }
 695         return mostRecentPrimary;
 696     }
 697
 698     private DroolsPdp getBestDesignated(Collection<DroolsPdp> pdps, boolean containsDesignated) {
 699         DroolsPdp mostRecentPrimary;
 700         mostRecentPrimary = null;
 701         for (DroolsPdp pdp : pdps) {
 702             if (mostRecentPrimary == null) {
 703                 mostRecentPrimary = pdp;
 704                 continue;
 705             }
 706             if (containsDesignated) { //Choose the site of the first designated date
 707                 if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) < 0) {
 708                     mostRecentPrimary = pdp;
 709                     logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 710                 }
 711             } else { //Choose the site with the latest designated date
 712                 if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 713                     mostRecentPrimary = pdp;
 714                     logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 715                 }
 716             }
 717         }
 718         return mostRecentPrimary;
 719     }
 720
 721     private DroolsPdp getLatestUndesignated(Collection<DroolsPdp> pdps, DroolsPdp mostRecentPrimary,
 722                     List<DroolsPdp> listOfDesignated) {
 723         for (DroolsPdp pdp : pdps) {
 724             if (listOfDesignated.contains(pdp)) {
 725                 continue; //Don't consider this entry
 726             }
 727             if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 728                 mostRecentPrimary = pdp;
 729                 logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 730             }
 731         }
 732         return mostRecentPrimary;
 733     }
 734
 735     private DroolsPdp getLatestDesignated(Collection<DroolsPdp> pdps, DroolsPdp mostRecentPrimary) {
 736         for (DroolsPdp pdp : pdps) {
 737             logger.debug("DesignatedWaiter.run pdp = {}"
 738                             + " pdp.getDesignatedDate() = {}",
 739                             pdp.getPdpId(), pdp.getDesignatedDate());
 740             if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 741                 mostRecentPrimary = pdp;
 742                 logger.debug(RUN_PRIMARY_MSG, mostRecentPrimary.getPdpId());
 743             }
 744         }
 745         return mostRecentPrimary;
 746     }
 747
 748     /**
 749      * Compue designated pdp.
 750      *
 751      * @param listOfDesignated list of designated pdps
 752      * @param mostRecentPrimary most recent primary pdpd
 753      * @return drools pdp object
 754      */
 755     public DroolsPdp computeDesignatedPdp(List<DroolsPdp> listOfDesignated, DroolsPdp mostRecentPrimary) {
 756         if (listOfDesignated.isEmpty()) {
 757             logger.debug("\nDesignatedWaiter.run: myPdp: {} listOfDesignated is: EMPTY.", myPdp.getPdpId());
 758             return null;
 759         }
 760
 761         if (listOfDesignated.size() == 1) {
 762             logger.debug("\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 763                             + "has ONE entry. PDP ID: {}", myPdp.getPdpId(), listOfDesignated.get(0).getPdpId());
 764             return listOfDesignated.get(0);
 765         }
 766
 767         logger.debug("DesignatedWaiter.run: myPdp: {} listOfDesignated.size(): {}", myPdp.getPdpId(),
 768                         listOfDesignated.size());
 769         DesignatedData data = new DesignatedData();
 770         for (DroolsPdp pdp : listOfDesignated) {
 771             DroolsPdp rejectedPdp;
 772
 773             // We need to determine if another PDP is the lowest priority
 774             if (nullSafeEquals(pdp.getSite(), mostRecentPrimary.getSite())) {
 775                 rejectedPdp = data.compareSameSite(pdp);
 776             } else {
 777                 rejectedPdp = data.compareDifferentSite(pdp);
 778             }
 779             // If the rejectedPdp is myPdp, we need to stand it down and demote it.  Each pdp is responsible
 780             // for demoting itself
 781             if (rejectedPdp != null && nullSafeEquals(rejectedPdp.getPdpId(),myPdp.getPdpId())) {
 782                 logger.debug("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated myPdp ID: {}"
 783                                 + " is NOT the lowest priority.  Executing stateManagement.demote()\n\n",
 784                                 myPdp.getPdpId(),
 785                                 myPdp.getPdpId());
 786                 // We found that myPdp is on the listOfDesignated and it is not the lowest priority
 787                 // So, we must demote it
 788                 demoteMyPdp();
 789             }
 790         } //end: for(DroolsPdp pdp : listOfDesignated)
 791
 792         DroolsPdp lowestPriorityPdp = data.getLowestPriority();
 793
 794         //now we have a valid value for lowestPriorityPdp
 795         logger.debug("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 796                         + "found the LOWEST priority pdp ID: {} "
 797                         + " It is now the designatedPpd from the perspective of myPdp ID: {} \n\n",
 798                         myPdp.getPdpId(), lowestPriorityPdp.getPdpId(), myPdp);
 799         return lowestPriorityPdp;
 800
 801     }
 802
 803     private class DesignatedData {
 804         private DroolsPdp lowestPrioritySameSite = null;
 805         private DroolsPdp lowestPriorityDifferentSite = null;
 806
 807         private DroolsPdp compareSameSite(DroolsPdp pdp) {
 808             if (lowestPrioritySameSite == null) {
 809                 if (lowestPriorityDifferentSite != null) {
 810                     //we need to reject lowestPriorityDifferentSite
 811                     DroolsPdp rejectedPdp = lowestPriorityDifferentSite;
 812                     lowestPriorityDifferentSite = pdp;
 813                     return rejectedPdp;
 814                 }
 815                 lowestPrioritySameSite = pdp;
 816                 return null;
 817             } else {
 818                 if (pdp.getPdpId().equals((lowestPrioritySameSite.getPdpId()))) {
 819                     return null;//nothing to compare
 820                 }
 821                 if (pdp.comparePriority(lowestPrioritySameSite) < 0) {
 822                     logger.debug("\nDesignatedWaiter.run: myPdp {}  listOfDesignated pdp ID: {}"
 823                                     + " has lower priority than pdp ID: {}",myPdp.getPdpId(), pdp.getPdpId(),
 824                                     lowestPrioritySameSite.getPdpId());
 825                     //we need to reject lowestPrioritySameSite
 826                     DroolsPdp rejectedPdp = lowestPrioritySameSite;
 827                     lowestPrioritySameSite = pdp;
 828                     return rejectedPdp;
 829                 } else {
 830                     //we need to reject pdp and keep lowestPrioritySameSite
 831                     logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {} "
 832                                     + " has higher priority than pdp ID: {}", myPdp.getPdpId(),pdp.getPdpId(),
 833                                     lowestPrioritySameSite.getPdpId());
 834                     return pdp;
 835                 }
 836             }
 837         }
 838
 839         private DroolsPdp compareDifferentSite(DroolsPdp pdp) {
 840             if (lowestPrioritySameSite != null) {
 841                 //if we already have a candidate for same site, we don't want to bother with different sites
 842                 return pdp;
 843             } else {
 844                 if (lowestPriorityDifferentSite == null) {
 845                     lowestPriorityDifferentSite = pdp;
 846                     return null;
 847                 }
 848                 if (pdp.getPdpId().equals((lowestPriorityDifferentSite.getPdpId()))) {
 849                     return null;//nothing to compare
 850                 }
 851                 if (pdp.comparePriority(lowestPriorityDifferentSite) < 0) {
 852                     logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 853                                     + " has lower priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 854                                     lowestPriorityDifferentSite.getPdpId());
 855                     //we need to reject lowestPriorityDifferentSite
 856                     DroolsPdp rejectedPdp = lowestPriorityDifferentSite;
 857                     lowestPriorityDifferentSite = pdp;
 858                     return rejectedPdp;
 859                 } else {
 860                     //we need to reject pdp and keep lowestPriorityDifferentSite
 861                     logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 862                                     + " has higher priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 863                                     lowestPriorityDifferentSite.getPdpId());
 864                     return pdp;
 865                 }
 866             }
 867         }
 868
 869         private DroolsPdp getLowestPriority() {
 870             return (lowestPrioritySameSite != null ? lowestPrioritySameSite : lowestPriorityDifferentSite);
 871         }
 872     }
 873
 874     private void demoteMyPdp() {
 875         try {
 876             //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 877             myPdp.setDesignated(false);
 878             pdpsConnector.setDesignated(myPdp, false);
 879             isDesignated = false;
 880             String standbyStatus = stateManagementFeature.getStandbyStatus();
 881             if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 882                     || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 883                 /*
 884                  * Only call demote if it is not already in the right state.  Don't worry about
 885                  * synching the lower level topic endpoint states.  That is done by the
 886                  * refreshStateAudit.
 887                  */
 888                 stateManagementFeature.demote();
 889             }
 890         } catch (Exception e) {
 891             myPdp.setDesignated(false);
 892             pdpsConnector.setDesignated(myPdp, false);
 893             isDesignated = false;
 894             logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 895                     + "demote myPdp {} myPdp.getPdpId(), message= {}", myPdp.getPdpId(),
 896                     e);
 897         }
 898     }
 899
 900     private class TimerUpdateClass extends TimerTask {
 901
 902         @Override
 903         public void run() {
 904             try {
 905                 logger.debug("TimerUpdateClass.run: entry");
 906                 checkWaitTimer();
 907             } catch (Exception e) {
 908                 logger.error("TimerUpdateClass.run caught an unexpected exception: ", e);
 909             }
 910             logger.debug("TimerUpdateClass.run.exit");
 911         }
 912     }
 913
 914     @Override
 915     public void checkThreadStatus() {
 916         checkWaitTimer();
 917     }
 918
 919     private void checkWaitTimer() {
 920         synchronized (checkWaitTimerLock) {
 921             try {
 922                 logger.debug("checkWaitTimer: entry");
 923                 Date now = new Date();
 924                 long nowMs = now.getTime();
 925                 long waitTimerMs = waitTimerLastRunDate.getTime();
 926
 927                 //give it 10 times leeway
 928                 if ((nowMs - waitTimerMs)  > 10 * pdpUpdateInterval) {
 929                     if (allSeemsWell == null || allSeemsWell) {
 930                         allSeemsWell = false;
 931                         logger.debug("checkWaitTimer: calling allSeemsWell with ALLNOTWELL param");
 932                         stateManagementFeature.allSeemsWell(this.getClass().getName(),
 933                                 StateManagementFeatureApiConstants.ALLNOTWELL_STATE,
 934                                 "DesignationWaiter/ElectionHandler has STALLED");
 935                     }
 936                     logger.error("checkWaitTimer: nowMs - waitTimerMs = {}"
 937                             + ", exceeds 10* pdpUpdateInterval = {}"
 938                             + " DesignationWaiter is STALLED!", (nowMs - waitTimerMs), (10 * pdpUpdateInterval));
 939                 } else if (allSeemsWell == null || !allSeemsWell) {
 940                     allSeemsWell = true;
 941                     stateManagementFeature.allSeemsWell(this.getClass().getName(),
 942                             StateManagementFeatureApiConstants.ALLSEEMSWELL_STATE,
 943                             "DesignationWaiter/ElectionHandler has RESUMED");
 944                     logger.info("DesignationWaiter/ElectionHandler has RESUMED");
 945                 }
 946                 logger.debug("checkWaitTimer: exit");
 947             } catch (Exception e) {
 948                 logger.error("checkWaitTimer: caught unexpected exception: ", e);
 949             }
 950         }
 951     }
 952
 953     private long getDWaiterStartMs() {
 954         Date now = new Date();
 955
 956         // Retrieve the ms since the epoch
 957         long nowMs = now.getTime();
 958
 959         // Time since the end of the last pdpUpdateInterval multiple
 960         long nowModMs = nowMs % pdpUpdateInterval;
 961
 962         // Time to the start of the next pdpUpdateInterval multiple
 963         long startMs = 2 * pdpUpdateInterval - nowModMs;
 964
 965         // Give the start time a minimum of a 5 second cushion
 966         if (startMs < 5000) {
 967             // Start at the beginning  of following interval
 968             startMs = pdpUpdateInterval + startMs;
 969         }
 970         return startMs;
 971     }
 972
 973     private boolean nullSafeEquals(Object one, Object two) {
 974         if (one == null && two == null) {
 975             return true;
 976         }
 977         if (one != null && two != null) {
 978             return one.equals(two);
 979         }
 980         return false;
 981     }
 982
 983     public String getPdpdNowActive() {
 984         return pdpdNowActive;
 985     }
 986
 987     public String getPdpdLastActive() {
 988         return pdpdLastActive;
 989     }
 990 }