gerrit.onap Code Review - policy/drools-pdp.git/blob

   1 /*
   2  * ============LICENSE_START=======================================================
   3  * feature-active-standby-management
   4  * ================================================================================
   5  * Copyright (C) 2017-2018 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.policy.drools.activestandby;
  22
  23 import java.util.ArrayList;
  24 import java.util.Collection;
  25 import java.util.Date;
  26 import java.util.List;
  27 import java.util.Timer;
  28 import java.util.TimerTask;
  29
  30 import org.onap.policy.common.im.StateManagement;
  31 import org.onap.policy.drools.statemanagement.StateManagementFeatureAPI;
  32 import org.slf4j.Logger;
  33 import org.slf4j.LoggerFactory;
  34
  35 public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
  36     // get an instance of logger
  37     private static final Logger  logger = LoggerFactory.getLogger(DroolsPdpsElectionHandler.class);
  38     private DroolsPdpsConnector pdpsConnector;
  39     private Object checkWaitTimerLock = new Object();
  40     private Object designationWaiterLock = new Object();
  41
  42     /*
  43      * Must be static, so it can be referenced by JpaDroolsPdpsConnector,
  44      * without requiring a reference to the election handler instantiation.
  45      */
  46     private static DroolsPdp myPdp;
  47
  48     private DesignationWaiter designationWaiter;
  49     private Timer updateWorker;
  50     private Timer waitTimer;
  51     private Date waitTimerLastRunDate;
  52
  53     // The interval between checks of the DesignationWaiter to be sure it is running.
  54     private int pdpCheckInterval;
  55
  56     // The interval between runs of the DesignationWaiter
  57     private int pdpUpdateInterval;
  58
  59     private volatile boolean isDesignated;
  60
  61     private String pdpdNowActive;
  62     private String pdpdLastActive;
  63
  64     /*
  65      * Start allSeemsWell with a value of null so that, on the first run
  66      * of the checkWaitTimer it will set the value in IntegrityMonitor
  67      * regardless of whether it needs to be set to true or false.
  68      */
  69     private Boolean allSeemsWell = null;
  70
  71     private StateManagementFeatureAPI stateManagementFeature;
  72
  73     private static boolean isUnitTesting = false;
  74     private static boolean isStalled = false;
  75
  76     /**
  77      * Constructor.
  78      *
  79      * @param pdps connectors
  80      * @param myPdp pdp
  81      */
  82     public DroolsPdpsElectionHandler(DroolsPdpsConnector pdps, DroolsPdp myPdp) {
  83         if (pdps == null) {
  84             logger.error("DroolsPdpsElectinHandler(): pdpsConnector==null");
  85             throw new IllegalArgumentException("DroolsPdpsElectinHandler(): pdpsConnector==null");
  86         }
  87         if (myPdp == null) {
  88             logger.error("DroolsPdpsElectinHandler(): droolsPdp==null");
  89             throw new IllegalArgumentException("DroolsPdpsElectinHandler(): DroolsPdp==null");
  90         }
  91
  92         pdpdNowActive = null;
  93         pdpdLastActive = null;
  94         this.pdpsConnector = pdps;
  95         DroolsPdpsElectionHandler.myPdp = myPdp;
  96         this.isDesignated = false;
  97         pdpCheckInterval = 3000;
  98         try {
  99             pdpCheckInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(
 100                     ActiveStandbyProperties.PDP_CHECK_INVERVAL));
 101         } catch (Exception e) {
 102             logger.error("Could not get pdpCheckInterval property. Using default {}",pdpCheckInterval, e);
 103         }
 104         pdpUpdateInterval = 2000;
 105         try {
 106             pdpUpdateInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(
 107                     ActiveStandbyProperties.PDP_UPDATE_INTERVAL));
 108         } catch (Exception e) {
 109             logger.error("Could not get pdpUpdateInterval property. Using default {} ", pdpUpdateInterval, e);
 110         }
 111
 112         Date now = new Date();
 113
 114         // Retrieve the ms since the epoch
 115         final long nowMs = now.getTime();
 116
 117         // Create the timer which will update the updateDate in DroolsPdpEntity table.
 118         // This is the heartbeat
 119         updateWorker = new Timer();
 120
 121         // Schedule the TimerUpdateClass to run at 100 ms and run at pdpCheckInterval ms thereafter
 122         // NOTE: The first run of the TimerUpdateClass results in myPdp being added to the
 123         // drools droolsPdpEntity table.
 124         updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
 125
 126         // Create the timer which will run the election algorithm
 127         waitTimer = new Timer();
 128
 129         // Schedule it to start in startMs ms
 130         // (so it will run after the updateWorker and run at pdpUpdateInterval ms thereafter
 131         long startMs = getDWaiterStartMs();
 132         designationWaiter = new DesignationWaiter();
 133         waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
 134         waitTimerLastRunDate = new Date(nowMs + startMs);
 135
 136         //Get the StateManagementFeature instance
 137
 138         for (StateManagementFeatureAPI feature : StateManagementFeatureAPI.impl.getList()) {
 139             if (feature.getResourceName().equals(myPdp.getPdpId())) {
 140                 if (logger.isDebugEnabled()) {
 141                     logger.debug("DroolsPdpsElectionHandler: Found StateManagementFeature"
 142                             + " with resourceName: {}", myPdp.getPdpId());
 143                 }
 144                 stateManagementFeature = feature;
 145                 break;
 146             }
 147         }
 148         if (stateManagementFeature == null) {
 149             logger.error("DroolsPdpsElectionHandler failed to initialize.  "
 150                     + "Unable to get instance of StateManagementFeatureAPI "
 151                     + "with resourceID: {}", myPdp.getPdpId());
 152         }
 153     }
 154
 155     public static void setIsUnitTesting(boolean val) {
 156         isUnitTesting = val;
 157     }
 158
 159     public static void setIsStalled(boolean val) {
 160         isStalled = val;
 161     }
 162
 163     /**
 164      * When the JpaDroolsPdpsConnector.standDown() method is invoked, it needs
 165      * access to myPdp, so it can keep its designation status in sync with the
 166      * DB.
 167      *
 168      * @param designated is designated value
 169      */
 170     public static void setMyPdpDesignated(boolean designated) {
 171         if (logger.isDebugEnabled()) {
 172             logger.debug("setMyPdpDesignated: designated= {}", designated);
 173         }
 174         myPdp.setDesignated(designated);
 175     }
 176
 177     private class DesignationWaiter extends TimerTask {
 178         // get an instance of logger
 179         private final Logger  logger = LoggerFactory.getLogger(DesignationWaiter.class);
 180
 181         @Override
 182         public void run() {
 183             try {
 184                 if (logger.isDebugEnabled()) {
 185                     logger.debug("DesignatedWaiter.run: Entering");
 186                 }
 187
 188                 //This is for testing the checkWaitTimer
 189                 if (isUnitTesting && isStalled) {
 190                     if (logger.isDebugEnabled()) {
 191                         logger.debug("DesignatedWaiter.run: isUnitTesting = {} isStalled = {}",
 192                                 isUnitTesting, isStalled);
 193                     }
 194                     return;
 195                 }
 196
 197                 synchronized (designationWaiterLock) {
 198
 199                     if (logger.isDebugEnabled()) {
 200                         logger.debug("DesignatedWaiter.run: Entering synchronized block");
 201                     }
 202
 203                     //It is possible that multiple PDPs are designated lead.  So, we will make a list of all designated
 204                     //PDPs and then decide which one really should be designated at the end.
 205                     List<DroolsPdp> listOfDesignated = new ArrayList<>();
 206
 207                     Collection<DroolsPdp> pdps = pdpsConnector.getDroolsPdps();
 208
 209                     if (logger.isDebugEnabled()) {
 210                         logger.debug("DesignatedWaiter.run: pdps.size= {}", pdps.size());
 211                     }
 212
 213                     //This is only true if all designated PDPs have failed
 214                     boolean designatedPdpHasFailed = pdpsConnector.hasDesignatedPdpFailed(pdps);
 215                     if (logger.isDebugEnabled()) {
 216                         logger.debug("DesignatedWaiter.run: designatedPdpHasFailed= {}", designatedPdpHasFailed);
 217                     }
 218                     for (DroolsPdp pdp : pdps) {
 219                         if (logger.isDebugEnabled()) {
 220                             logger.debug("DesignatedWaiter.run: evaluating pdp ID: {}", pdp.getPdpId());
 221                         }
 222
 223                         /*
 224                          * Note: side effect of isPdpCurrent is that any stale but
 225                          * designated PDPs will be marked as un-designated.
 226                          */
 227                         boolean isCurrent = pdpsConnector.isPdpCurrent(pdp);
 228
 229                         /*
 230                          * We can't use stateManagement.getStandbyStatus() here, because
 231                          * we need the standbyStatus, not for this PDP, but for the PDP
 232                          * being processed by this loop iteration.
 233                          */
 234                         String standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 235                         if (standbyStatus == null) {
 236                             // Treat this case as a cold standby -- if we
 237                             // abort here, no sessions will be created in a
 238                             // single-node test environment.
 239                             standbyStatus = StateManagement.COLD_STANDBY;
 240                         }
 241                         if (logger.isDebugEnabled()) {
 242                             logger.debug("DesignatedWaiter.run: PDP= {},  isCurrent= {}", pdp.getPdpId(), isCurrent);
 243                         }
 244
 245                         /*
 246                          * There are 4 combinations of isDesignated and isCurrent.  We will examine each one in-turn
 247                          * and evaluate the each pdp in the list of pdps against each combination.
 248                          *
 249                          * This is the first combination of isDesignated and isCurrent
 250                          */
 251                         if (pdp.isDesignated()  &&  isCurrent) {
 252                             //It is current, but it could have a standbystatus=coldstandby / hotstandby
 253                             //If so, we need to stand it down and demote it
 254                             if (!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)) {
 255                                 if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 256                                     if (logger.isDebugEnabled()) {
 257                                         logger.debug("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 258                                                 + "butstandbystatus is not providingservice. "
 259                                                 + " Executing stateManagement.demote()" + "\n\n", myPdp.getPdpId());
 260                                     }
 261                                     // So, we must demote it
 262                                     try {
 263                                         //Keep the order like this.  StateManagement is last since it
 264                                         //triggers controller shutdown
 265                                         //This will change isDesignated and it can enter another if(combination) below
 266                                         pdpsConnector.standDownPdp(pdp.getPdpId());
 267                                         myPdp.setDesignated(false);
 268                                         isDesignated = false;
 269                                         if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 270                                                 || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 271                                             /*
 272                                              * Only demote it if it appears it has not already been demoted. Don't worry
 273                                              * about synching with the topic endpoint states.  That is done by the
 274                                              * refreshStateAudit
 275                                              */
 276                                             stateManagementFeature.demote();
 277                                         }
 278                                         //update the standbystatus to check in a later
 279                                         //combination of isDesignated and isCurrent
 280                                         standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 281                                     } catch (Exception e) {
 282                                         logger.error("DesignatedWaiter.run: myPdp: {} "
 283                                                 + "Caught Exception attempting to demote myPdp,"
 284                                                 + "message= {}", myPdp.getPdpId(), e);
 285                                     }
 286                                 } else {
 287                                     // Don't demote a remote PDP that is current.  It should catch itself
 288                                     if (logger.isDebugEnabled()) {
 289                                         logger.debug("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 290                                                 + "but standbystatus is not providingservice. "
 291                                                 + " Cannot execute stateManagement.demote() "
 292                                                 + "since it it is not myPdp\n\n",
 293                                                 myPdp.getPdpId());
 294                                     }
 295                                 }
 296
 297                             } else {
 298                                 // If we get here, it is ok to be on the list
 299                                 if (logger.isDebugEnabled()) {
 300                                     logger.debug("DesignatedWaiter.run: PDP= {} is designated, "
 301                                             + "current and {} Noting PDP as "
 302                                             + "designated, standbyStatus= {}",
 303                                             pdp.getPdpId(), standbyStatus, standbyStatus);
 304                                 }
 305                                 listOfDesignated.add(pdp);
 306                             }
 307
 308
 309                         }
 310
 311
 312                         /*
 313                          * The second combination of isDesignated and isCurrent
 314                          *
 315                          * PDP is designated but not current; it has failed.
 316                          * So we stand it down (it doesn't matter what
 317                          * its standbyStatus is). None of these go on the list.
 318                          */
 319                         if (pdp.isDesignated()  &&  !isCurrent) {
 320                             if (logger.isDebugEnabled()) {
 321                                 logger.debug("INFO: DesignatedWaiter.run: PDP= {} is currently "
 322                                         + "designated but is not current; "
 323                                         + "it has failed.  Standing down.  standbyStatus= {}",
 324                                         pdp.getPdpId(), standbyStatus);
 325                             }
 326                             /*
 327                              * Changes designated to 0 but it is still potentially providing service
 328                              * Will affect isDesignated, so, it can enter an if(combination) below
 329                              */
 330                             pdpsConnector.standDownPdp(pdp.getPdpId());
 331
 332                             //need to change standbystatus to coldstandby
 333                             if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 334                                 if (logger.isDebugEnabled()) {
 335                                     logger.debug("\n\nDesignatedWaiter.run: myPdp {} is not Current. "
 336                                             + " Executing stateManagement.disableFailed()\n\n", myPdp.getPdpId());
 337                                 }
 338                                 // We found that myPdp is designated but not current
 339                                 // So, we must cause it to disableFail
 340                                 try {
 341                                     myPdp.setDesignated(false);
 342                                     pdpsConnector.setDesignated(myPdp, false);
 343                                     isDesignated = false;
 344                                     stateManagementFeature.disableFailed();
 345                                 } catch (Exception e) {
 346                                     logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception "
 347                                             + "attempting to disableFail myPdp {}, message= {}",
 348                                             myPdp.getPdpId(), myPdp.getPdpId(), e);
 349                                 }
 350                             } else { //it is a remote PDP that is failed
 351                                 if (logger.isDebugEnabled()) {
 352                                     logger.debug("\n\nDesignatedWaiter.run: PDP {} is not Current. "
 353                                             + " Executing stateManagement.disableFailed(otherResourceName)\n\n",
 354                                             pdp.getPdpId() );
 355                                 }
 356                                 // We found a PDP is designated but not current
 357                                 // We already called standdown(pdp) which will change designated to false
 358                                 // Now we need to disableFail it to get its states in synch.  The standbyStatus
 359                                 // should equal coldstandby
 360                                 try {
 361                                     stateManagementFeature.disableFailed(pdp.getPdpId());
 362                                 } catch (Exception e) {
 363                                     logger.error("DesignatedWaiter.run: for PDP {}  Caught Exception attempting to "
 364                                             + "disableFail({}), message= {}",
 365                                             pdp.getPdpId(), pdp.getPdpId(), e);
 366                                 }
 367
 368                             }
 369                             continue; //we are not going to do anything else with this pdp
 370                         }
 371
 372                         /*
 373                          * The third combination of isDesignated and isCurrent
 374                          * /*
 375                          * If a PDP is not currently designated but is providing service
 376                          * (erroneous, but recoverable) or hot standby
 377                          * we can add it to the list of possible designated if all the designated have failed
 378                          */
 379                         if (!pdp.isDesignated() && isCurrent) {
 380                             if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 381                                     || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 382                                 if (logger.isDebugEnabled()) {
 383                                     logger.debug("\n\nDesignatedWaiter.run: PDP {}"
 384                                             + " is NOT designated but IS current and"
 385                                             + " has a standbystatus= {}", pdp.getPdpId(), standbyStatus);
 386                                 }
 387                                 // Since it is current, we assume it can adjust its own state.
 388                                 // We will demote if it is myPdp
 389                                 if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 390                                     //demote it
 391                                     if (logger.isDebugEnabled()) {
 392                                         logger.debug("DesignatedWaiter.run: PDP {} going to "
 393                                                 + "setDesignated = false and calling stateManagement.demote",
 394                                                 pdp.getPdpId());
 395                                     }
 396                                     try {
 397                                         //Keep the order like this.
 398                                         //StateManagement is last since it triggers controller shutdown
 399                                         pdpsConnector.setDesignated(myPdp, false);
 400                                         myPdp.setDesignated(false);
 401                                         isDesignated = false;
 402                                         //This is definitely not a redundant call.
 403                                         //It is attempting to correct a problem
 404                                         stateManagementFeature.demote();
 405                                         //recheck the standbystatus
 406                                         standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 407                                     } catch (Exception e) {
 408                                         logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception "
 409                                                 + "attempting to demote myPdp {}, message = {}",  myPdp.getPdpId(),
 410                                                 myPdp.getPdpId(), e);
 411                                     }
 412
 413                                 }
 414                             }
 415                             if (standbyStatus.equals(StateManagement.HOT_STANDBY) && designatedPdpHasFailed) {
 416                                 //add it to the list
 417                                 if (logger.isDebugEnabled()) {
 418                                     logger.debug("INFO: DesignatedWaiter.run: PDP= {}"
 419                                             + " is not designated but is {} and designated PDP "
 420                                             + "has failed.  standbyStatus= {}", pdp.getPdpId(),
 421                                             standbyStatus, standbyStatus);
 422                                 }
 423                                 listOfDesignated.add(pdp);
 424                             }
 425                             continue; //done with this one
 426                         }
 427
 428                         /*
 429                          * The fourth combination of isDesignated and isCurrent
 430                          *
 431                          * We are not going to put any of these on the list since it appears they have failed.
 432
 433                          *
 434                          */
 435                         if (!pdp.isDesignated() && !isCurrent) {
 436                             if (logger.isDebugEnabled()) {
 437                                 logger.debug("INFO: DesignatedWaiter.run: PDP= {} "
 438                                         + "designated= {}, current= {}, "
 439                                         + "designatedPdpHasFailed= {}, "
 440                                         + "standbyStatus= {}",pdp.getPdpId(),
 441                                         pdp.isDesignated(), isCurrent, designatedPdpHasFailed, standbyStatus);
 442                             }
 443                             if (!standbyStatus.equals(StateManagement.COLD_STANDBY)) {
 444                                 //stand it down
 445                                 //disableFail it
 446                                 pdpsConnector.standDownPdp(pdp.getPdpId());
 447                                 if (pdp.getPdpId().equals(myPdp.getPdpId())) {
 448                                     /*
 449                                      * I don't actually know how this condition could happen,
 450                                      * but if it did, we would want
 451                                      * to declare it failed.
 452                                      */
 453                                     if (logger.isDebugEnabled()) {
 454                                         logger.debug("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 455                                                 + " Executing stateManagement.disableFailed()\n\n", myPdp.getPdpId());
 456                                     }
 457                                     // So, we must disableFail it
 458                                     try {
 459                                         //Keep the order like this.
 460                                         //StateManagement is last since it triggers controller shutdown
 461                                         pdpsConnector.setDesignated(myPdp, false);
 462                                         myPdp.setDesignated(false);
 463                                         isDesignated = false;
 464                                         stateManagementFeature.disableFailed();
 465                                     } catch (Exception e) {
 466                                         logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 467                                                 + "disableFail myPdp {}, message= {}",
 468                                                 myPdp.getPdpId(), myPdp.getPdpId(), e);
 469                                     }
 470                                 } else { //it is remote
 471                                     if (logger.isDebugEnabled()) {
 472                                         logger.debug("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 473                                                 + " Executing stateManagement.disableFailed({})\n\n",
 474                                                 myPdp.getPdpId(), pdp.getPdpId());
 475                                     }
 476                                     // We already called standdown(pdp) which will change designated to false
 477                                     // Now we need to disableFail it to get its states in sync.
 478                                     // StandbyStatus = coldstandby
 479                                     try {
 480                                         stateManagementFeature.disableFailed(pdp.getPdpId());
 481                                     } catch (Exception e) {
 482                                         logger.error("DesignatedWaiter.run: for PDP {}"
 483                                                 + " Caught Exception attempting to disableFail({})"
 484                                                 + ", message=", pdp.getPdpId(), pdp.getPdpId(), e);
 485                                     }
 486                                 }
 487                             }
 488                         }
 489
 490
 491                     } // end pdps loop
 492
 493                     /*
 494                      * We have checked the four combinations of isDesignated and isCurrent.  Where appropriate,
 495                      * we added the PDPs to the potential list of designated pdps
 496                      *
 497                      * We need to give priority to pdps on the same site that is currently being used
 498                      * First, however, we must sanitize the list of designated to make sure their are
 499                      * only designated members or non-designated members.  There should not be both in
 500                      * the list. Because there are real time delays, it is possible that both types could
 501                      * be on the list.
 502                      */
 503
 504                     listOfDesignated = santizeDesignatedList(listOfDesignated);
 505
 506                     /*
 507                      * We need to figure out the last pdp that was the primary so we can get the last site
 508                      * name and the last session numbers.  We need to create a "dummy" droolspdp since
 509                      * it will be used in later comparisons and cannot be null.
 510                      */
 511
 512                     DroolsPdp mostRecentPrimary = computeMostRecentPrimary(pdps, listOfDesignated);
 513
 514                     if (mostRecentPrimary != null) {
 515                         pdpdLastActive = mostRecentPrimary.getPdpId();
 516                     }
 517
 518
 519                     /*
 520                      * It is possible to get here with more than one pdp designated and providingservice. This normally
 521                      * occurs when there is a race condition with multiple nodes coming up at the same time. If that is
 522                      * the case we must determine which one is the one that should be designated and which one should
 523                      * be demoted.
 524                      *
 525                      * It is possible to have 0, 1, 2 or more but not all, or all designated.
 526                      *   If we have one designated and current, we chose it and are done
 527                      *   If we have 2 or more, but not all, we must determine which one is in the same site as
 528                      *   the previously designated pdp.
 529                      */
 530                     DroolsPdp designatedPdp = computeDesignatedPdp(listOfDesignated, mostRecentPrimary);
 531                     if (designatedPdp != null) {
 532                         pdpdNowActive = designatedPdp.getPdpId();
 533                     }
 534
 535                     if (designatedPdp == null) {
 536                         logger.warn("WARNING: DesignatedWaiter.run: No viable PDP found to be Designated. "
 537                             + "designatedPdp still null.");
 538                         // Just to be sure the parameters are correctly set
 539                         myPdp.setDesignated(false);
 540                         pdpsConnector.setDesignated(myPdp,false);
 541                         isDesignated = false;
 542
 543                         waitTimerLastRunDate = new Date();
 544                         if (logger.isDebugEnabled()) {
 545                             logger.debug("DesignatedWaiter.run (designatedPdp == null) waitTimerLastRunDate = {}",
 546                                     waitTimerLastRunDate);
 547                         }
 548                         myPdp.setUpdatedDate(waitTimerLastRunDate);
 549                         pdpsConnector.update(myPdp);
 550
 551                         return;
 552
 553                     } else if (designatedPdp.getPdpId().equals(myPdp.getPdpId())) {
 554                         if (logger.isDebugEnabled()) {
 555                             logger.debug("DesignatedWaiter.run: designatedPdp is PDP={}", myPdp.getPdpId());
 556                         }
 557                         /*
 558                          * update function expects myPdp.isDesignated to be true.
 559                          */
 560                         try {
 561                             //Keep the order like this.  StateManagement is last since it triggers controller init
 562                             myPdp.setDesignated(true);
 563                             myPdp.setDesignatedDate(new Date());
 564                             pdpsConnector.setDesignated(myPdp, true);
 565                             isDesignated = true;
 566                             String standbyStatus = stateManagementFeature.getStandbyStatus();
 567                             if (!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)) {
 568                                 /*
 569                                  * Only call promote if it is not already in the right state.  Don't worry about
 570                                  * synching the lower level topic endpoint states.  That is done by the
 571                                  * refreshStateAudit.
 572                                  * Note that we need to fetch the session list from 'mostRecentPrimary'
 573                                  * at this point -- soon, 'mostRecentPrimary' will be set to this host.
 574                                  */
 575                                 //this.sessions = mostRecentPrimary.getSessions();
 576                                 stateManagementFeature.promote();
 577                             }
 578                         } catch (Exception e) {
 579                             logger.error("ERROR: DesignatedWaiter.run: Caught Exception attempting to promote PDP={}"
 580                                     + ", message=", myPdp.getPdpId(), e);
 581                             myPdp.setDesignated(false);
 582                             pdpsConnector.setDesignated(myPdp,false);
 583                             isDesignated = false;
 584                             //If you can't promote it, demote it
 585                             try {
 586                                 String standbyStatus = stateManagementFeature.getStandbyStatus();
 587                                 if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 588                                         || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 589                                     /*
 590                                      * Only call demote if it is not already in the right state.  Don't worry about
 591                                      * synching the lower level topic endpoint states.  That is done by the
 592                                      * refreshStateAudit.
 593                                      */
 594                                     stateManagementFeature.demote();
 595                                 }
 596                             } catch (Exception e1) {
 597                                 logger.error("ERROR: DesignatedWaiter.run: Caught StandbyStatusException "
 598                                         + "attempting to promote then demote PDP={}, message=",
 599                                         myPdp.getPdpId(), e1);
 600                             }
 601
 602                         }
 603                         waitTimerLastRunDate = new Date();
 604                         if (logger.isDebugEnabled()) {
 605                             logger.debug("DesignatedWaiter.run (designatedPdp.getPdpId().equals(myPdp.getPdpId())) "
 606                                     + "waitTimerLastRunDate = " + waitTimerLastRunDate);
 607                         }
 608                         myPdp.setUpdatedDate(waitTimerLastRunDate);
 609                         pdpsConnector.update(myPdp);
 610
 611                         return;
 612                     }
 613                     isDesignated = false;
 614
 615                 } // end synchronized
 616                 if (logger.isDebugEnabled()) {
 617                     logger.debug("DesignatedWaiter.run: myPdp: {}; Returning, isDesignated= {}",
 618                             isDesignated, myPdp.getPdpId());
 619                 }
 620
 621                 Date tmpDate = new Date();
 622                 if (logger.isDebugEnabled()) {
 623                     logger.debug("DesignatedWaiter.run (end of run) waitTimerLastRunDate = {}", tmpDate);
 624                 }
 625
 626                 waitTimerLastRunDate = tmpDate;
 627                 myPdp.setUpdatedDate(waitTimerLastRunDate);
 628                 pdpsConnector.update(myPdp);
 629
 630             } catch (Exception e) {
 631                 logger.error("DesignatedWaiter.run caught an unexpected exception: ", e);
 632             }
 633         } // end run
 634     }
 635
 636     /**
 637      * Sanitize designated list.
 638      *
 639      * @param listOfDesignated list of designated pdps
 640      * @return
 641      */
 642     public List<DroolsPdp> santizeDesignatedList(List<DroolsPdp> listOfDesignated) {
 643
 644         boolean containsDesignated = false;
 645         boolean containsHotStandby = false;
 646         List<DroolsPdp> listForRemoval = new ArrayList<>();
 647         for (DroolsPdp pdp : listOfDesignated) {
 648             if (logger.isDebugEnabled()) {
 649                 logger.debug("DesignatedWaiter.run sanitizing: pdp = {}"
 650                         + " isDesignated = {}",pdp.getPdpId(), pdp.isDesignated());
 651             }
 652             if (pdp.isDesignated()) {
 653                 containsDesignated = true;
 654             } else {
 655                 containsHotStandby = true;
 656                 listForRemoval.add(pdp);
 657             }
 658         }
 659         if (containsDesignated && containsHotStandby) {
 660             //remove the hot standby from the list
 661             listOfDesignated.removeAll(listForRemoval);
 662         }
 663         return listOfDesignated;
 664     }
 665
 666     /**
 667      * Compute most recent primary.
 668      *
 669      * @param pdps collection of pdps
 670      * @param listOfDesignated list of designated pdps
 671      * @return
 672      */
 673     public DroolsPdp computeMostRecentPrimary(Collection<DroolsPdp> pdps, List<DroolsPdp> listOfDesignated) {
 674         boolean containsDesignated = false;
 675         for (DroolsPdp pdp : listOfDesignated) {
 676             if (pdp.isDesignated()) {
 677                 containsDesignated = true;
 678             }
 679         }
 680         DroolsPdp mostRecentPrimary = new DroolsPdpImpl(null, true, 1, new Date(0));
 681         mostRecentPrimary.setSiteName(null);
 682         if (logger.isDebugEnabled()) {
 683             logger.debug("DesignatedWaiter.run listOfDesignated.size() = {}", listOfDesignated.size());
 684         }
 685         if (listOfDesignated.size() <= 1) {
 686             if (logger.isDebugEnabled()) {
 687                 logger.debug("DesignatedWainter.run: listOfDesignated.size <=1");
 688             }
 689             //Only one or none is designated or hot standby.  Choose the latest designated date
 690             for (DroolsPdp pdp : pdps) {
 691                 if (logger.isDebugEnabled()) {
 692                     logger.debug("DesignatedWaiter.run pdp = {}"
 693                             + " pdp.getDesignatedDate() = {}",
 694                             pdp.getPdpId(), pdp.getDesignatedDate());
 695                 }
 696                 if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 697                     mostRecentPrimary = pdp;
 698                     if (logger.isDebugEnabled()) {
 699                         logger.debug("DesignatedWaiter.run mostRecentPrimary = {}",
 700                                 mostRecentPrimary.getPdpId());
 701                     }
 702                 }
 703             }
 704         } else if (listOfDesignated.size() == pdps.size()) {
 705             if (logger.isDebugEnabled()) {
 706                 logger.debug("DesignatedWainter.run: listOfDesignated.size = pdps.size() which is {}", pdps.size());
 707             }
 708             //They are all designated or all hot standby.
 709             mostRecentPrimary = null;
 710             for (DroolsPdp pdp : pdps) {
 711                 if (mostRecentPrimary == null) {
 712                     mostRecentPrimary = pdp;
 713                     continue;
 714                 }
 715                 if (containsDesignated) { //Choose the site of the first designated date
 716                     if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) < 0) {
 717                         mostRecentPrimary = pdp;
 718                         if (logger.isDebugEnabled()) {
 719                             logger.debug("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 720                         }
 721                     }
 722                 } else { //Choose the site with the latest designated date
 723                     if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 724                         mostRecentPrimary = pdp;
 725                         if (logger.isDebugEnabled()) {
 726                             logger.debug("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 727                         }
 728                     }
 729                 }
 730             }
 731         } else {
 732             if (logger.isDebugEnabled()) {
 733                 logger.debug("DesignatedWainter.run: Some but not all are designated or hot standby. ");
 734             }
 735             //Some but not all are designated or hot standby.
 736             if (containsDesignated) {
 737                 if (logger.isDebugEnabled()) {
 738                     logger.debug("DesignatedWainter.run: containsDesignated = {}", containsDesignated);
 739                 }
 740                 /*
 741                  * The list only contains designated.  This is a problem.  It is most likely a race
 742                  * condition that resulted in two thinking they should be designated. Choose the
 743                  * site with the latest designated date for the pdp not included on the designated list.
 744                  * This should be the site that had the last designation before this race condition
 745                  * occurred.
 746                  */
 747                 for (DroolsPdp pdp : pdps) {
 748                     if (listOfDesignated.contains(pdp)) {
 749                         continue; //Don't consider this entry
 750                     }
 751                     if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 752                         mostRecentPrimary = pdp;
 753                         if (logger.isDebugEnabled()) {
 754                             logger.debug("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 755                         }
 756                     }
 757                 }
 758             } else {
 759                 if (logger.isDebugEnabled()) {
 760                     logger.debug("DesignatedWainter.run: containsDesignated = {}", containsDesignated);
 761                 }
 762                 //The list only contains hot standby. Choose the site of the latest designated date
 763                 for (DroolsPdp pdp : pdps) {
 764                     if (pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0) {
 765                         mostRecentPrimary = pdp;
 766                         if (logger.isDebugEnabled()) {
 767                             logger.debug("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 768                         }
 769                     }
 770                 }
 771             }
 772         }
 773         return mostRecentPrimary;
 774     }
 775
 776     /**
 777      * Compue designated pdp.
 778      *
 779      * @param listOfDesignated list of designated pdps
 780      * @param mostRecentPrimary most recent primary pdpd
 781      * @return
 782      */
 783     public DroolsPdp computeDesignatedPdp(List<DroolsPdp> listOfDesignated, DroolsPdp mostRecentPrimary) {
 784         DroolsPdp designatedPdp = null;
 785         DroolsPdp lowestPriorityPdp = null;
 786         if (listOfDesignated.size() > 1) {
 787             if (logger.isDebugEnabled()) {
 788                 logger.debug("DesignatedWaiter.run: myPdp: {} listOfDesignated.size(): {}", myPdp.getPdpId(),
 789                         listOfDesignated.size());
 790             }
 791             DroolsPdp rejectedPdp = null;
 792             DroolsPdp lowestPrioritySameSite = null;
 793             DroolsPdp lowestPriorityDifferentSite = null;
 794             for (DroolsPdp pdp : listOfDesignated) {
 795                 // We need to determine if another PDP is the lowest priority
 796                 if (nullSafeEquals(pdp.getSiteName(),mostRecentPrimary.getSiteName())) {
 797                     if (lowestPrioritySameSite == null) {
 798                         if (lowestPriorityDifferentSite != null) {
 799                             rejectedPdp = lowestPriorityDifferentSite;
 800                         }
 801                         lowestPrioritySameSite = pdp;
 802                     } else {
 803                         if (pdp.getPdpId().equals((lowestPrioritySameSite.getPdpId()))) {
 804                             continue;//nothing to compare
 805                         }
 806                         if (pdp.comparePriority(lowestPrioritySameSite) < 0) {
 807                             if (logger.isDebugEnabled()) {
 808                                 logger.debug("\nDesignatedWaiter.run: myPdp {}  listOfDesignated pdp ID: {}"
 809                                         + " has lower priority than pdp ID: {}",myPdp.getPdpId(), pdp.getPdpId(),
 810                                         lowestPrioritySameSite.getPdpId());
 811                             }
 812                             //we need to reject lowestPrioritySameSite
 813                             rejectedPdp = lowestPrioritySameSite;
 814                             lowestPrioritySameSite = pdp;
 815                         } else {
 816                             //we need to reject pdp and keep lowestPrioritySameSite
 817                             if (logger.isDebugEnabled()) {
 818                                 logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {} "
 819                                         + " has higher priority than pdp ID: {}", myPdp.getPdpId(),pdp.getPdpId(),
 820                                         lowestPrioritySameSite.getPdpId());
 821                             }
 822                             rejectedPdp = pdp;
 823                         }
 824                     }
 825                 } else {
 826                     if (lowestPrioritySameSite != null) {
 827                         //if we already have a candidate for same site, we don't want to bother with different sites
 828                         rejectedPdp = pdp;
 829                     } else {
 830                         if (lowestPriorityDifferentSite == null) {
 831                             lowestPriorityDifferentSite = pdp;
 832                             continue;
 833                         }
 834                         if (pdp.getPdpId().equals((lowestPriorityDifferentSite.getPdpId()))) {
 835                             continue;//nothing to compare
 836                         }
 837                         if (pdp.comparePriority(lowestPriorityDifferentSite) < 0) {
 838                             if (logger.isDebugEnabled()) {
 839                                 logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 840                                         + " has lower priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 841                                         lowestPriorityDifferentSite.getPdpId());
 842                             }
 843                             //we need to reject lowestPriorityDifferentSite
 844                             rejectedPdp = lowestPriorityDifferentSite;
 845                             lowestPriorityDifferentSite = pdp;
 846                         } else {
 847                             //we need to reject pdp and keep lowestPriorityDifferentSite
 848                             if (logger.isDebugEnabled()) {
 849                                 logger.debug("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 850                                         + " has higher priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 851                                         lowestPriorityDifferentSite.getPdpId());
 852                             }
 853                             rejectedPdp = pdp;
 854                         }
 855                     }
 856                 }
 857                 // If the rejectedPdp is myPdp, we need to stand it down and demote it.  Each pdp is responsible
 858                 // for demoting itself
 859                 if (rejectedPdp != null && nullSafeEquals(rejectedPdp.getPdpId(),myPdp.getPdpId())) {
 860                     if (logger.isDebugEnabled()) {
 861                         logger.debug("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated myPdp ID: {}"
 862                                 + " is NOT the lowest priority.  Executing stateManagement.demote()\n\n",
 863                                 myPdp.getPdpId(),
 864                                 myPdp.getPdpId());
 865                     }
 866                     // We found that myPdp is on the listOfDesignated and it is not the lowest priority
 867                     // So, we must demote it
 868                     try {
 869                         //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 870                         myPdp.setDesignated(false);
 871                         pdpsConnector.setDesignated(myPdp, false);
 872                         isDesignated = false;
 873                         String standbyStatus = stateManagementFeature.getStandbyStatus();
 874                         if (!(standbyStatus.equals(StateManagement.HOT_STANDBY)
 875                                 || standbyStatus.equals(StateManagement.COLD_STANDBY))) {
 876                             /*
 877                              * Only call demote if it is not already in the right state.  Don't worry about
 878                              * synching the lower level topic endpoint states.  That is done by the
 879                              * refreshStateAudit.
 880                              */
 881                             stateManagementFeature.demote();
 882                         }
 883                     } catch (Exception e) {
 884                         myPdp.setDesignated(false);
 885                         pdpsConnector.setDesignated(myPdp, false);
 886                         isDesignated = false;
 887                         logger.error("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 888                                 + "demote myPdp {} myPdp.getPdpId(), message= {}", myPdp.getPdpId(),
 889                                 e);
 890                     }
 891                 }
 892             } //end: for(DroolsPdp pdp : listOfDesignated)
 893             if (lowestPrioritySameSite != null) {
 894                 lowestPriorityPdp = lowestPrioritySameSite;
 895             } else {
 896                 lowestPriorityPdp = lowestPriorityDifferentSite;
 897             }
 898             //now we have a valid value for lowestPriorityPdp
 899             if (logger.isDebugEnabled()) {
 900                 logger.debug("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 901                         + "found the LOWEST priority pdp ID: {} "
 902                         + " It is now the designatedPpd from the perspective of myPdp ID: {} \n\n",
 903                         myPdp.getPdpId(), lowestPriorityPdp.getPdpId(), myPdp);
 904             }
 905             designatedPdp = lowestPriorityPdp;
 906
 907         } else if (listOfDesignated.isEmpty()) {
 908             if (logger.isDebugEnabled()) {
 909                 logger.debug("\nDesignatedWaiter.run: myPdp: {} listOfDesignated is: EMPTY.", myPdp.getPdpId());
 910             }
 911             designatedPdp = null;
 912         } else { //only one in listOfDesignated
 913             if (logger.isDebugEnabled()) {
 914                 logger.debug("\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 915                         + "has ONE entry. PDP ID: {}", myPdp.getPdpId(), listOfDesignated.get(0).getPdpId());
 916             }
 917             designatedPdp = listOfDesignated.get(0);
 918         }
 919         return designatedPdp;
 920
 921     }
 922
 923     private class TimerUpdateClass extends TimerTask {
 924
 925         @Override
 926         public void run() {
 927             try {
 928                 if (logger.isDebugEnabled()) {
 929                     logger.debug("TimerUpdateClass.run: entry");
 930                 }
 931                 checkWaitTimer();
 932             } catch (Exception e) {
 933                 logger.error("TimerUpdateClass.run caught an unexpected exception: ", e);
 934             }
 935             if (logger.isDebugEnabled()) {
 936                 logger.debug("TimerUpdateClass.run.exit");
 937             }
 938         }
 939     }
 940
 941     @Override
 942     public void checkThreadStatus() {
 943         checkWaitTimer();
 944     }
 945
 946     private void checkWaitTimer() {
 947         synchronized (checkWaitTimerLock) {
 948             try {
 949                 if (logger.isDebugEnabled()) {
 950                     logger.debug("checkWaitTimer: entry");
 951                 }
 952                 Date now = new Date();
 953                 long nowMs = now.getTime();
 954                 long waitTimerMs = waitTimerLastRunDate.getTime();
 955
 956                 //give it 10 times leeway
 957                 if ((nowMs - waitTimerMs)  > 10 * pdpUpdateInterval) {
 958                     if (allSeemsWell == null || allSeemsWell) {
 959                         allSeemsWell = false;
 960                         if (logger.isDebugEnabled()) {
 961                             logger.debug("checkWaitTimer: calling allSeemsWell with ALLNOTWELL param");
 962                         }
 963                         stateManagementFeature.allSeemsWell(this.getClass().getName(),
 964                                 StateManagementFeatureAPI.ALLNOTWELL_STATE,
 965                                 "DesignationWaiter/ElectionHandler has STALLED");
 966                     }
 967                     logger.error("checkWaitTimer: nowMs - waitTimerMs = {}"
 968                             + ", exceeds 10* pdpUpdateInterval = {}"
 969                             + " DesignationWaiter is STALLED!", (nowMs - waitTimerMs), (10 * pdpUpdateInterval));
 970                 } else if (allSeemsWell == null || !allSeemsWell) {
 971                     allSeemsWell = true;
 972                     stateManagementFeature.allSeemsWell(this.getClass().getName(),
 973                             StateManagementFeatureAPI.ALLSEEMSWELL_STATE,
 974                             "DesignationWaiter/ElectionHandler has RESUMED");
 975                     logger.info("DesignationWaiter/ElectionHandler has RESUMED");
 976                 }
 977                 if (logger.isDebugEnabled()) {
 978                     logger.debug("checkWaitTimer: exit");
 979                 }
 980             } catch (Exception e) {
 981                 logger.error("checkWaitTimer: caught unexpected exception: ", e);
 982             }
 983         }
 984     }
 985
 986     private long getDWaiterStartMs() {
 987         Date now = new Date();
 988
 989         // Retrieve the ms since the epoch
 990         long nowMs = now.getTime();
 991
 992         // Time since the end of the last pdpUpdateInterval multiple
 993         long nowModMs = nowMs % pdpUpdateInterval;
 994
 995         // Time to the start of the next pdpUpdateInterval multiple
 996         long startMs = 2 * pdpUpdateInterval - nowModMs;
 997
 998         // Give the start time a minimum of a 5 second cushion
 999         if (startMs < 5000) {
1000             // Start at the beginning  of following interval
1001             startMs = pdpUpdateInterval + startMs;
1002         }
1003         return startMs;
1004     }
1005
1006     private boolean nullSafeEquals(Object one, Object two) {
1007         if (one == null && two == null) {
1008             return true;
1009         }
1010         if (one != null && two != null) {
1011             return one.equals(two);
1012         }
1013         return false;
1014     }
1015
1016     public String getPdpdNowActive() {
1017         return pdpdNowActive;
1018     }
1019
1020     public String getPdpdLastActive() {
1021         return pdpdLastActive;
1022     }
1023 }