gerrit.onap Code Review - policy/drools-pdp.git/blob

   1 /*
   2  * ============LICENSE_START=======================================================
   3  * feature-active-standby-management
   4  * ================================================================================
   5  * Copyright (C) 2017-2018 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.policy.drools.activestandby;
  22
  23 import java.util.ArrayList;
  24 import java.util.Collection;
  25 import java.util.Date;
  26 import java.util.List;
  27 import java.util.Timer;
  28 import java.util.TimerTask;
  29
  30 import org.onap.policy.common.im.StateManagement;
  31 import org.slf4j.Logger;
  32 import org.slf4j.LoggerFactory;
  33 import org.onap.policy.drools.statemanagement.StateManagementFeatureAPI;
  34
  35 public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
  36         // get an instance of logger
  37         private static final Logger  logger = LoggerFactory.getLogger(DroolsPdpsElectionHandler.class);
  38         private DroolsPdpsConnector pdpsConnector;
  39         private Object checkWaitTimerLock = new Object();
  40         private Object designationWaiterLock = new Object();
  41
  42         /*
  43          * Must be static, so it can be referenced by JpaDroolsPdpsConnector,
  44          * without requiring a reference to the election handler instantiation.
  45          */
  46         private static DroolsPdp myPdp;
  47
  48         private DesignationWaiter designationWaiter;
  49         private Timer updateWorker;
  50         private Timer waitTimer;
  51         private Date waitTimerLastRunDate;
  52
  53         // The interval between checks of the DesignationWaiter to be sure it is running.
  54         private int pdpCheckInterval;
  55
  56         // The interval between runs of the DesignationWaiter
  57         private int pdpUpdateInterval;
  58
  59         private volatile boolean isDesignated;
  60
  61         private String pdpdNowActive;
  62         private String pdpdLastActive;
  63
  64         /*
  65          * Start allSeemsWell with a value of null so that, on the first run
  66          * of the checkWaitTimer it will set the value in IntegrityMonitor
  67          * regardless of whether it needs to be set to true or false.
  68          */
  69         private Boolean allSeemsWell=null;
  70
  71         private StateManagementFeatureAPI stateManagementFeature;
  72
  73         private static boolean isUnitTesting = false;
  74         private static boolean isStalled = false;
  75
  76         public DroolsPdpsElectionHandler(DroolsPdpsConnector pdps, DroolsPdp myPdp){
  77                 if (pdps == null) {
  78                         logger.error("DroolsPdpsElectinHandler(): pdpsConnector==null");
  79                         throw new IllegalArgumentException("DroolsPdpsElectinHandler(): pdpsConnector==null");
  80                 }
  81                 if (myPdp == null){
  82                         logger.error("DroolsPdpsElectinHandler(): droolsPdp==null");
  83                         throw new IllegalArgumentException("DroolsPdpsElectinHandler(): DroolsPdp==null");
  84                 }
  85
  86                 pdpdNowActive = null;
  87                 pdpdLastActive = null;
  88                 this.pdpsConnector = pdps;
  89                 DroolsPdpsElectionHandler.myPdp = myPdp;
  90                 this.isDesignated = false;
  91                 pdpCheckInterval = 3000;
  92                 try{
  93                         pdpCheckInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(ActiveStandbyProperties.PDP_CHECK_INVERVAL));
  94                 }catch(Exception e){
  95                         logger.error
  96                         ("Could not get pdpCheckInterval property. Using default {}",pdpCheckInterval, e);
  97                 }
  98                 pdpUpdateInterval = 2000;
  99                 try{
 100                         pdpUpdateInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(ActiveStandbyProperties.PDP_UPDATE_INTERVAL));
 101                 }catch(Exception e){
 102                         logger.error
 103                         ("Could not get pdpUpdateInterval property. Using default {} ", pdpUpdateInterval, e);
 104                 }
 105
 106                 Date now = new Date();
 107
 108                 // Retrieve the ms since the epoch
 109                 long nowMs = now.getTime();
 110
 111                 // Create the timer which will update the updateDate in DroolsPdpEntity table.
 112                 // This is the heartbeat
 113                 updateWorker = new Timer();
 114
 115                 // Schedule the TimerUpdateClass to run at 100 ms and run at pdpCheckInterval ms thereafter
 116                 // NOTE: The first run of the TimerUpdateClass results in myPdp being added to the
 117                 // drools droolsPdpEntity table.
 118                 updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
 119
 120                 // Create the timer which will run the election algorithm
 121                 waitTimer = new Timer();
 122
 123                 // Schedule it to start in startMs ms (so it will run after the updateWorker and run at pdpUpdateInterval ms thereafter
 124                 long startMs = getDWaiterStartMs();
 125                 designationWaiter = new DesignationWaiter();
 126                 waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
 127                 waitTimerLastRunDate = new Date(nowMs + startMs);
 128
 129                 //Get the StateManagementFeature instance
 130
 131                 for (StateManagementFeatureAPI feature : StateManagementFeatureAPI.impl.getList())
 132                 {
 133                         if (feature.getResourceName().equals(myPdp.getPdpId()))
 134                         {
 135                                 if(logger.isDebugEnabled()){
 136                                         logger.debug("DroolsPdpsElectionHandler: Found StateManagementFeature"
 137                                                 + " with resourceName: {}", myPdp.getPdpId());
 138                                 }
 139                                 stateManagementFeature = feature;
 140                                 break;
 141                         }
 142                 }
 143                 if(stateManagementFeature == null){
 144                         logger.error("DroolsPdpsElectionHandler failed to initialize.  "
 145                                         + "Unable to get instance of StateManagementFeatureAPI "
 146                                         + "with resourceID: {}", myPdp.getPdpId());
 147                 }
 148         }
 149
 150         public static void setIsUnitTesting(boolean val){
 151                 isUnitTesting = val;
 152         }
 153         public static void setIsStalled(boolean val){
 154                 isStalled = val;
 155         }
 156
 157         /*
 158          * When the JpaDroolsPdpsConnector.standDown() method is invoked, it needs
 159          * access to myPdp, so it can keep its designation status in sync with the
 160          * DB.
 161          */
 162         public static void setMyPdpDesignated(boolean designated) {
 163                 if(logger.isDebugEnabled()){
 164                         logger.debug
 165                         ("setMyPdpDesignated: designated= {}", designated);
 166                 }
 167                 myPdp.setDesignated(designated);
 168         }
 169
 170         private class DesignationWaiter extends TimerTask {
 171                 // get an instance of logger
 172                 private final Logger  logger = LoggerFactory.getLogger(DesignationWaiter.class);
 173
 174                 @Override
 175                 public void run() {
 176                         try{
 177                                 if(logger.isDebugEnabled()){
 178                                         logger.debug
 179                                         ("DesignatedWaiter.run: Entering");
 180                                 }
 181
 182                                 //This is for testing the checkWaitTimer
 183                                 if(isUnitTesting && isStalled){
 184                                         if(logger.isDebugEnabled()){
 185                                                 logger.debug("DesignatedWaiter.run: isUnitTesting = {} isStalled = {}", isUnitTesting, isStalled);
 186                                         }
 187                                         return;
 188                                 }
 189
 190                                 synchronized (designationWaiterLock) {
 191
 192                                         if(logger.isDebugEnabled()){
 193                                                 logger.debug
 194                                                 ("DesignatedWaiter.run: Entering synchronized block");
 195                                         }
 196
 197                                         //It is possible that multiple PDPs are designated lead.  So, we will make a list of all designated
 198                                         //PDPs and then decide which one really should be designated at the end.
 199                                         List<DroolsPdp> listOfDesignated = new ArrayList<>();
 200
 201                                         Collection<DroolsPdp> pdps = pdpsConnector.getDroolsPdps();
 202                                         DroolsPdp designatedPdp = null;
 203
 204                                         if(logger.isDebugEnabled()){
 205                                                 logger.debug
 206                                                 ("DesignatedWaiter.run: pdps.size= {}", pdps.size());
 207                                         }
 208
 209                                         //This is only true if all designated PDPs have failed
 210                                         boolean designatedPdpHasFailed = pdpsConnector.hasDesignatedPdpFailed(pdps);
 211                                         if(logger.isDebugEnabled()){
 212                                                 logger.debug
 213                                                 ("DesignatedWaiter.run: designatedPdpHasFailed= {}", designatedPdpHasFailed);
 214                                         }
 215                                         for (DroolsPdp pdp : pdps) {
 216                                                 if(logger.isDebugEnabled()){
 217                                                         logger.debug
 218                                                         ("DesignatedWaiter.run: evaluating pdp ID: {}", pdp.getPdpId());
 219                                                 }
 220
 221                                                 /*
 222                                                  * Note: side effect of isPdpCurrent is that any stale but
 223                                                  * designated PDPs will be marked as un-designated.
 224                                                  */
 225                                                 boolean isCurrent = pdpsConnector.isPdpCurrent(pdp);
 226
 227                                                 /*
 228                                                  * We can't use stateManagement.getStandbyStatus() here, because
 229                                                  * we need the standbyStatus, not for this PDP, but for the PDP
 230                                                  * being processed by this loop iteration.
 231                                                  */
 232                                                 String standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 233                                                 if(standbyStatus==null){
 234                                                         // Treat this case as a cold standby -- if we
 235                                                         // abort here, no sessions will be created in a
 236                                                         // single-node test environment.
 237                                                         standbyStatus = StateManagement.COLD_STANDBY;
 238                                                 }
 239                                                 if(logger.isDebugEnabled()){
 240                                                         logger.debug
 241                                                         ("DesignatedWaiter.run: PDP= {},  isCurrent= {}", pdp.getPdpId(), isCurrent);
 242                                                 }
 243
 244                                                 /*
 245                                                  * There are 4 combinations of isDesignated and isCurrent.  We will examine each one in-turn
 246                                                  * and evaluate the each pdp in the list of pdps against each combination.
 247                                                  *
 248                                                  * This is the first combination of isDesignated and isCurrent
 249                                                  */
 250                                                 if (pdp.isDesignated()  &&  isCurrent) {
 251                                                         //It is current, but it could have a standbystatus=coldstandby / hotstandby
 252                                                         //If so, we need to stand it down and demote it
 253                                                         if(!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)){
 254                                                                 if(pdp.getPdpId().equals(myPdp.getPdpId())){
 255                                                                         if(logger.isDebugEnabled()){
 256                                                                                 logger.debug
 257                                                                                 ("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 258                                                                                         + "butstandbystatus is not providingservice. "
 259                                                                                         + " Executing stateManagement.demote()" + "\n\n", myPdp.getPdpId());
 260                                                                         }
 261                                                                         // So, we must demote it
 262                                                                         try {
 263                                                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 264                                                                                 //This will change isDesignated and it can enter another if(combination) below
 265                                                                                 pdpsConnector.standDownPdp(pdp.getPdpId());
 266                                                                                 myPdp.setDesignated(false);
 267                                                                                 isDesignated = false;
 268                                                                                 if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 269                                                                                                 standbyStatus.equals(StateManagement.COLD_STANDBY))){
 270                                                                                         /*
 271                                                                                          * Only demote it if it appears it has not already been demoted. Don't worry
 272                                                                                          * about synching with the topic endpoint states.  That is done by the
 273                                                                                          * refreshStateAudit
 274                                                                                          */
 275                                                                                         stateManagementFeature.demote();
 276                                                                                 }
 277                                                                                 //update the standbystatus to check in a later combination of isDesignated and isCurrent
 278                                                                                 standbyStatus=stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 279                                                                         } catch (Exception e) {
 280                                                                                 logger.error
 281                                                                                 ("DesignatedWaiter.run: myPdp: {} "
 282                                                                                                 + "Caught Exception attempting to demote myPdp,"
 283                                                                                                 + "message= {}", myPdp.getPdpId(), e);
 284                                                                         }
 285                                                                 }else{
 286                                                                         // Don't demote a remote PDP that is current.  It should catch itself
 287                                                                         if(logger.isDebugEnabled()){
 288                                                                                 logger.debug
 289                                                                                 ("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 290                                                                                         + "but standbystatus is not providingservice. "
 291                                                                                         + " Cannot execute stateManagement.demote() since it it is not myPdp\n\n", myPdp.getPdpId());
 292                                                                         }
 293                                                                 }
 294
 295                                                         }else{
 296                                                                 // If we get here, it is ok to be on the list
 297                                                                 if(logger.isDebugEnabled()){
 298                                                                         logger.debug
 299                                                                         ("DesignatedWaiter.run: PDP= {} is designated, current and {} Noting PDP as "
 300                                                                                 + "designated, standbyStatus= {}", pdp.getPdpId(), standbyStatus, standbyStatus);
 301                                                                 }
 302                                                                 listOfDesignated.add(pdp);
 303                                                         }
 304
 305
 306                                                 }
 307
 308
 309                                                 /*
 310                                                  * The second combination of isDesignated and isCurrent
 311                                                  *
 312                                                  * PDP is designated but not current; it has failed.   So we stand it down (it doesn't matter what
 313                                                  * its standbyStatus is). None of these go on the list.
 314                                                  */
 315                                                 if (pdp.isDesignated()  &&  !isCurrent) {
 316                                                         if(logger.isDebugEnabled()){
 317                                                                 logger.debug
 318                                                                 ("INFO: DesignatedWaiter.run: PDP= {} is currently designated but is not current; "
 319                                                                         + "it has failed.  Standing down.  standbyStatus= {}", pdp.getPdpId(), standbyStatus);
 320                                                         }
 321                                                         /*
 322                                                          * Changes designated to 0 but it is still potentially providing service
 323                                                          * Will affect isDesignated, so, it can enter an if(combination) below
 324                                                          */
 325                                                         pdpsConnector.standDownPdp(pdp.getPdpId());
 326
 327                                                         //need to change standbystatus to coldstandby
 328                                                         if (pdp.getPdpId().equals(myPdp.getPdpId())){
 329                                                                 if(logger.isDebugEnabled()){
 330                                                                         logger.debug
 331                                                                         ("\n\nDesignatedWaiter.run: myPdp {} is not Current. "
 332                                                                                 + " Executing stateManagement.disableFailed()\n\n", myPdp.getPdpId());
 333                                                                 }
 334                                                                 // We found that myPdp is designated but not current
 335                                                                 // So, we must cause it to disableFail
 336                                                                 try {
 337                                                                         myPdp.setDesignated(false);
 338                                                                         pdpsConnector.setDesignated(myPdp, false);
 339                                                                         isDesignated = false;
 340                                                                         stateManagementFeature.disableFailed();
 341                                                                 } catch (Exception e) {
 342                                                                         logger.error
 343                                                                         ("DesignatedWaiter.run: myPdp: {} Caught Exception "
 344                                                                                         + "attempting to disableFail myPdp {}, message= {}",
 345                                                                                         myPdp.getPdpId(), myPdp.getPdpId(), e);
 346                                                                 }
 347                                                         } else { //it is a remote PDP that is failed
 348                                                                 if(logger.isDebugEnabled()){
 349                                                                         logger.debug
 350                                                                         ("\n\nDesignatedWaiter.run: PDP {} is not Current. "
 351                                                                                 + " Executing stateManagement.disableFailed(otherResourceName)\n\n", pdp.getPdpId() );
 352                                                                 }
 353                                                                 // We found a PDP is designated but not current
 354                                                                 // We already called standdown(pdp) which will change designated to false
 355                                                                 // Now we need to disableFail it to get its states in synch.  The standbyStatus
 356                                                                 // should equal coldstandby
 357                                                                 try {
 358                                                                         stateManagementFeature.disableFailed(pdp.getPdpId());
 359                                                                 } catch (Exception e) {
 360                                                                         logger.error
 361                                                                         ("DesignatedWaiter.run: for PDP {}  Caught Exception attempting to "
 362                                                                                         + "disableFail({}), message= {}",
 363                                                                                         pdp.getPdpId(), pdp.getPdpId(), e);
 364                                                                 }
 365
 366                                                         }
 367                                                         continue; //we are not going to do anything else with this pdp
 368                                                 }
 369
 370                                                 /*
 371                                                  * The third combination of isDesignated and isCurrent
 372                                                  * /*
 373                                                  * If a PDP is not currently designated but is providing service (erroneous, but recoverable) or hot standby
 374                                                  * we can add it to the list of possible designated if all the designated have failed
 375                                                  */
 376                                                 if (!pdp.isDesignated() && isCurrent){
 377                                                         if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 378                                                                         standbyStatus.equals(StateManagement.COLD_STANDBY))){
 379                                                                 if(logger.isDebugEnabled()){
 380                                                                         logger.debug("\n\nDesignatedWaiter.run: PDP {}"
 381                                                                                 + " is NOT designated but IS current and"
 382                                                                                 + " has a standbystatus= {}", pdp.getPdpId(), standbyStatus);
 383                                                                 }
 384                                                                 // Since it is current, we assume it can adjust its own state.
 385                                                                 // We will demote if it is myPdp
 386                                                                 if(pdp.getPdpId().equals(myPdp.getPdpId())){
 387                                                                         //demote it
 388                                                                         if(logger.isDebugEnabled()){
 389                                                                                 logger.debug("DesignatedWaiter.run: PDP {} going to "
 390                                                                                         + "setDesignated = false and calling stateManagement.demote",  pdp.getPdpId());
 391                                                                         }
 392                                                                         try {
 393                                                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 394                                                                                 pdpsConnector.setDesignated(myPdp, false);
 395                                                                                 myPdp.setDesignated(false);
 396                                                                                 isDesignated = false;
 397                                                                                 //This is definitely not a redundant call.  It is attempting to correct a problem
 398                                                                                 stateManagementFeature.demote();
 399                                                                                 //recheck the standbystatus
 400                                                                                 standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 401                                                                         } catch (Exception e) {
 402                                                                                 logger.error
 403                                                                                 ("DesignatedWaiter.run: myPdp: {} Caught Exception "
 404                                                                                                 + "attempting to demote myPdp {}, message = {}",  myPdp.getPdpId(),
 405                                                                                                 myPdp.getPdpId(), e);
 406                                                                         }
 407
 408                                                                 }
 409                                                         }
 410                                                         if(standbyStatus.equals(StateManagement.HOT_STANDBY) && designatedPdpHasFailed){
 411                                                                 //add it to the list
 412                                                                 if(logger.isDebugEnabled()){
 413                                                                         logger.debug
 414                                                                         ("INFO: DesignatedWaiter.run: PDP= {}"
 415                                                                                 + " is not designated but is {} and designated PDP "
 416                                                                                 + "has failed.  standbyStatus= {}", pdp.getPdpId(),
 417                                                                                 standbyStatus, standbyStatus);
 418                                                                 }
 419                                                                 listOfDesignated.add(pdp);
 420                                                         }
 421                                                         continue; //done with this one
 422                                                 }
 423
 424                                                 /*
 425                                                  * The fourth combination of isDesignated and isCurrent
 426                                                  *
 427                                                  * We are not going to put any of these on the list since it appears they have failed.
 428
 429                                                  *
 430                                                  */
 431                                                 if(!pdp.isDesignated() && !isCurrent) {
 432                                                         if(logger.isDebugEnabled()){
 433                                                                 logger.debug
 434                                                                 ("INFO: DesignatedWaiter.run: PDP= {} "
 435                                                                         + "designated= {}, current= {}, "
 436                                                                         + "designatedPdpHasFailed= {}, "
 437                                                                         + "standbyStatus= {}",pdp.getPdpId(),
 438                                                                         pdp.isDesignated(), isCurrent, designatedPdpHasFailed, standbyStatus);
 439                                                         }
 440                                                         if(!standbyStatus.equals(StateManagement.COLD_STANDBY)){
 441                                                                 //stand it down
 442                                                                 //disableFail it
 443                                                                 pdpsConnector.standDownPdp(pdp.getPdpId());
 444                                                                 if(pdp.getPdpId().equals(myPdp.getPdpId())){
 445                                                                         /*
 446                                                                          * I don't actually know how this condition could happen, but if it did, we would want
 447                                                                          * to declare it failed.
 448                                                                          */
 449                                                                         if(logger.isDebugEnabled()){
 450                                                                                 logger.debug
 451                                                                                 ("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 452                                                                                         + " Executing stateManagement.disableFailed()\n\n", myPdp.getPdpId());
 453                                                                         }
 454                                                                         // So, we must disableFail it
 455                                                                         try {
 456                                                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 457                                                                                 pdpsConnector.setDesignated(myPdp, false);
 458                                                                                 myPdp.setDesignated(false);
 459                                                                                 isDesignated = false;
 460                                                                                 stateManagementFeature.disableFailed();
 461                                                                         } catch (Exception e) {
 462                                                                                 logger.error
 463                                                                                 ("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 464                                                                                                 + "disableFail myPdp {}, message= {}",
 465                                                                                                  myPdp.getPdpId(), myPdp.getPdpId(), e);
 466                                                                         }
 467                                                                 }else{//it is remote
 468                                                                         if(logger.isDebugEnabled()){
 469                                                                                 logger.debug
 470                                                                                 ("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 471                                                                                         + " Executing stateManagement.disableFailed({})\n\n",
 472                                                                                         myPdp.getPdpId(), pdp.getPdpId());
 473                                                                         }
 474                                                                         // We already called standdown(pdp) which will change designated to false
 475                                                                         // Now we need to disableFail it to get its states in sync.  StandbyStatus = coldstandby
 476                                                                         try {
 477                                                                                 stateManagementFeature.disableFailed(pdp.getPdpId());
 478                                                                         } catch (Exception e) {
 479                                                                                 logger.error
 480                                                                                 ("DesignatedWaiter.run: for PDP {}"
 481                                                                                                 + " Caught Exception attempting to disableFail({})"
 482                                                                                                 + ", message=", pdp.getPdpId(), pdp.getPdpId(), e);
 483                                                                         }
 484                                                                 }
 485                                                         }
 486                                                 }
 487
 488
 489                                         } // end pdps loop
 490
 491                                         /*
 492                                          * We have checked the four combinations of isDesignated and isCurrent.  Where appropriate,
 493                                          * we added the PDPs to the potential list of designated pdps
 494                                          *
 495                                          * We need to give priority to pdps on the same site that is currently being used
 496                                          * First, however, we must sanitize the list of designated to make sure their are
 497                                          * only designated members or non-designated members.  There should not be both in
 498                                          * the list. Because there are real time delays, it is possible that both types could
 499                                          * be on the list.
 500                                          */
 501
 502                                         listOfDesignated = santizeDesignatedList(listOfDesignated);
 503
 504                                         /*
 505                                          * We need to figure out the last pdp that was the primary so we can get the last site
 506                                          * name and the last session numbers.  We need to create a "dummy" droolspdp since
 507                                          * it will be used in later comparisons and cannot be null.
 508                                          */
 509
 510                                         DroolsPdp mostRecentPrimary = computeMostRecentPrimary(pdps, listOfDesignated);
 511
 512                                         if(mostRecentPrimary != null){
 513                                                 pdpdLastActive = mostRecentPrimary.getPdpId();
 514                                         }
 515
 516
 517                                         /*
 518                                          * It is possible to get here with more than one pdp designated and providingservice. This normally
 519                                          * occurs when there is a race condition with multiple nodes coming up at the same time. If that is
 520                                          * the case we must determine which one is the one that should be designated and which one should
 521                                          * be demoted.
 522                                          *
 523                                          * It is possible to have 0, 1, 2 or more but not all, or all designated.
 524                                          *   If we have one designated and current, we chose it and are done
 525                                          *   If we have 2 or more, but not all, we must determine which one is in the same site as
 526                                          *   the previously designated pdp.
 527                                          */
 528
 529                                         designatedPdp = computeDesignatedPdp(listOfDesignated, mostRecentPrimary);
 530                                         if(designatedPdp != null){
 531                                                 pdpdNowActive = designatedPdp.getPdpId();
 532                                         }
 533
 534                                         if (designatedPdp == null) {
 535                                                 logger.warn
 536                                                 ("WARNING: DesignatedWaiter.run: No viable PDP found to be Designated. designatedPdp still null.");
 537                                                 // Just to be sure the parameters are correctly set
 538                                                 myPdp.setDesignated(false);
 539                                                 pdpsConnector.setDesignated(myPdp,false);
 540                                                 isDesignated = false;
 541
 542                                                 waitTimerLastRunDate = new Date();
 543                                                 if(logger.isDebugEnabled()){
 544                                                         logger.debug("DesignatedWaiter.run (designatedPdp == null) waitTimerLastRunDate = {}", waitTimerLastRunDate);
 545                                                 }
 546                                                 myPdp.setUpdatedDate(waitTimerLastRunDate);
 547                                                 pdpsConnector.update(myPdp);
 548
 549                                                 return;
 550
 551                                         } else if (designatedPdp.getPdpId().equals(myPdp.getPdpId())) {
 552                                                 if(logger.isDebugEnabled()){
 553                                                         logger.debug
 554                                                         ("DesignatedWaiter.run: designatedPdp is PDP={}", myPdp.getPdpId());
 555                                                 }
 556                                                 /*
 557                                                  * update function expects myPdp.isDesignated to be true.
 558                                                  */
 559                                                 try {
 560                                                         //Keep the order like this.  StateManagement is last since it triggers controller init
 561                                                         myPdp.setDesignated(true);
 562                                                         myPdp.setDesignatedDate(new Date());
 563                                                         pdpsConnector.setDesignated(myPdp, true);
 564                                                         isDesignated = true;
 565                                                         String standbyStatus = stateManagementFeature.getStandbyStatus();
 566                                                         if(!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)){
 567                                                                 /*
 568                                                                  * Only call promote if it is not already in the right state.  Don't worry about
 569                                                                  * synching the lower level topic endpoint states.  That is done by the
 570                                                                  * refreshStateAudit.
 571                                                                  * Note that we need to fetch the session list from 'mostRecentPrimary'
 572                                                                  * at this point -- soon, 'mostRecentPrimary' will be set to this host.
 573                                                                  */
 574                                                                 //this.sessions = mostRecentPrimary.getSessions();
 575                                                                 stateManagementFeature.promote();
 576                                                         }
 577                                                 } catch (Exception e) {
 578                                                         logger.error
 579                                                         ("ERROR: DesignatedWaiter.run: Caught Exception attempting to promote PDP={}"
 580                                                                         + ", message=", myPdp.getPdpId(), e);
 581                                                         myPdp.setDesignated(false);
 582                                                         pdpsConnector.setDesignated(myPdp,false);
 583                                                         isDesignated = false;
 584                                                         //If you can't promote it, demote it
 585                                                         try {
 586                                                                 String standbyStatus = stateManagementFeature.getStandbyStatus();
 587                                                                 if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 588                                                                                 standbyStatus.equals(StateManagement.COLD_STANDBY))){
 589                                                                         /*
 590                                                                          * Only call demote if it is not already in the right state.  Don't worry about
 591                                                                          * synching the lower level topic endpoint states.  That is done by the
 592                                                                          * refreshStateAudit.
 593                                                                          */
 594                                                                         stateManagementFeature.demote();
 595                                                                 }
 596                                                         } catch (Exception e1) {
 597                                                                 logger.error
 598                                                                 ("ERROR: DesignatedWaiter.run: Caught StandbyStatusException "
 599                                                                                 + "attempting to promote then demote PDP={}, message=",
 600                                                                                 myPdp.getPdpId(), e1);
 601                                                         }
 602
 603                                                 }
 604                                                 waitTimerLastRunDate = new Date();
 605                                                 if(logger.isDebugEnabled()){
 606                                                         logger.debug("DesignatedWaiter.run (designatedPdp.getPdpId().equals(myPdp.getPdpId())) "
 607                                                                 + "waitTimerLastRunDate = " + waitTimerLastRunDate);
 608                                                 }
 609                                                 myPdp.setUpdatedDate(waitTimerLastRunDate);
 610                                                 pdpsConnector.update(myPdp);
 611
 612                                                 return;
 613                                         }
 614                                         isDesignated = false;
 615
 616                                 } // end synchronized
 617                                 if(logger.isDebugEnabled()){
 618                                         logger.debug
 619                                         ("DesignatedWaiter.run: myPdp: {}; Returning, isDesignated= {}",
 620                                                 isDesignated, myPdp.getPdpId());
 621                                 }
 622
 623                                 Date tmpDate = new Date();
 624                                 if(logger.isDebugEnabled()){
 625                                         logger.debug("DesignatedWaiter.run (end of run) waitTimerLastRunDate = {}", tmpDate);
 626                                 }
 627
 628                                 waitTimerLastRunDate = tmpDate;
 629                                 myPdp.setUpdatedDate(waitTimerLastRunDate);
 630                                 pdpsConnector.update(myPdp);
 631
 632                         }catch(Exception e){
 633                                 logger.error("DesignatedWaiter.run caught an unexpected exception: ", e);
 634                         }
 635                 } // end run
 636         }
 637
 638         public List<DroolsPdp> santizeDesignatedList(List<DroolsPdp> listOfDesignated){
 639
 640                 boolean containsDesignated = false;
 641                 boolean containsHotStandby = false;
 642                 List<DroolsPdp> listForRemoval = new ArrayList<>();
 643                 for(DroolsPdp pdp : listOfDesignated){
 644                         if(logger.isDebugEnabled()){
 645                                 logger.debug
 646                                 ("DesignatedWaiter.run sanitizing: pdp = {}"
 647                                         + " isDesignated = {}",pdp.getPdpId(), pdp.isDesignated());
 648                         }
 649                         if(pdp.isDesignated()){
 650                                 containsDesignated = true;
 651                         }else {
 652                                 containsHotStandby = true;
 653                                 listForRemoval.add(pdp);
 654                         }
 655                 }
 656                 if(containsDesignated && containsHotStandby){
 657                         //remove the hot standby from the list
 658                         listOfDesignated.removeAll(listForRemoval);
 659                 }
 660                 return listOfDesignated;
 661         }
 662
 663         public DroolsPdp computeMostRecentPrimary(Collection<DroolsPdp> pdps, List<DroolsPdp> listOfDesignated){
 664                 boolean containsDesignated = false;
 665                 for(DroolsPdp pdp : listOfDesignated){
 666                         if(pdp.isDesignated()){
 667                                 containsDesignated = true;
 668                         }
 669                 }
 670                 DroolsPdp mostRecentPrimary = new DroolsPdpImpl(null, true, 1, new Date(0));
 671                 mostRecentPrimary.setSiteName(null);
 672                 if(logger.isDebugEnabled()){
 673                         logger.debug
 674                         ("DesignatedWaiter.run listOfDesignated.size() = {}", listOfDesignated.size());
 675                 }
 676                 if(listOfDesignated.size() <=1){
 677                         if(logger.isDebugEnabled()){
 678                                 logger.debug("DesignatedWainter.run: listOfDesignated.size <=1");
 679                         }
 680                         //Only one or none is designated or hot standby.  Choose the latest designated date
 681                         for(DroolsPdp pdp : pdps){
 682                                 if(logger.isDebugEnabled()){
 683                                         logger.debug
 684                                         ("DesignatedWaiter.run pdp = {}"
 685                                                 + " pdp.getDesignatedDate() = {}", pdp.getPdpId(), pdp.getDesignatedDate());
 686                                 }
 687                                 if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 688                                         mostRecentPrimary = pdp;
 689                                         if(logger.isDebugEnabled()){
 690                                                 logger.debug
 691                                                 ("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 692                                         }
 693                                 }
 694                         }
 695                 }else if(listOfDesignated.size() == pdps.size()){
 696                         if(logger.isDebugEnabled()){
 697                                 logger.debug("DesignatedWainter.run: listOfDesignated.size = pdps.size() which is {}", pdps.size());
 698                         }
 699                         //They are all designated or all hot standby.
 700                         mostRecentPrimary = null;
 701                         for(DroolsPdp pdp : pdps){
 702                                 if(mostRecentPrimary == null){
 703                                         mostRecentPrimary = pdp;
 704                                         continue;
 705                                 }
 706                                 if(containsDesignated){ //Choose the site of the first designated date
 707                                         if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) < 0){
 708                                                 mostRecentPrimary = pdp;
 709                                                 if(logger.isDebugEnabled()){
 710                                                         logger.debug
 711                                                         ("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 712                                                 }
 713                                         }
 714                                 }else{ //Choose the site with the latest designated date
 715                                         if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 716                                                 mostRecentPrimary = pdp;
 717                                                 if(logger.isDebugEnabled()){
 718                                                         logger.debug
 719                                                         ("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 720                                                 }
 721                                         }
 722                                 }
 723                         }
 724                 }else{
 725                         if(logger.isDebugEnabled()){
 726                                 logger.debug("DesignatedWainter.run: Some but not all are designated or hot standby. ");
 727                         }
 728                         //Some but not all are designated or hot standby.
 729                         if(containsDesignated){
 730                                 if(logger.isDebugEnabled()){
 731                                         logger.debug("DesignatedWainter.run: containsDesignated = {}", containsDesignated);
 732                                 }
 733                                 /*
 734                                  * The list only contains designated.  This is a problem.  It is most likely a race
 735                                  * condition that resulted in two thinking they should be designated. Choose the
 736                                  * site with the latest designated date for the pdp not included on the designated list.
 737                                  * This should be the site that had the last designation before this race condition
 738                                  * occurred.
 739                                  */
 740                                 for(DroolsPdp pdp : pdps){
 741                                         if(listOfDesignated.contains(pdp)){
 742                                                 continue; //Don't consider this entry
 743                                         }
 744                                         if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 745                                                 mostRecentPrimary = pdp;
 746                                                 if(logger.isDebugEnabled()){
 747                                                         logger.debug
 748                                                         ("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 749                                                 }
 750                                         }
 751                                 }
 752                         }else{
 753                                 if(logger.isDebugEnabled()){
 754                                         logger.debug("DesignatedWainter.run: containsDesignated = {}", containsDesignated);
 755                                 }
 756                                 //The list only contains hot standby. Choose the site of the latest designated date
 757                                 for(DroolsPdp pdp : pdps){
 758                                         if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 759                                                 mostRecentPrimary = pdp;
 760                                                 if(logger.isDebugEnabled()){
 761                                                         logger.debug
 762                                                         ("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 763                                                 }
 764                                         }
 765                                 }
 766                         }
 767                 }
 768                 return mostRecentPrimary;
 769         }
 770
 771         public DroolsPdp computeDesignatedPdp(List<DroolsPdp> listOfDesignated, DroolsPdp mostRecentPrimary){
 772                 DroolsPdp designatedPdp = null;
 773                 DroolsPdp lowestPriorityPdp = null;
 774                 if(listOfDesignated.size() > 1){
 775                         if(logger.isDebugEnabled()){
 776                                 logger.debug
 777                                 ("DesignatedWaiter.run: myPdp: {} listOfDesignated.size(): {}", myPdp.getPdpId(), listOfDesignated.size());
 778                         }
 779                         DroolsPdp rejectedPdp = null;
 780                         DroolsPdp lowestPrioritySameSite = null;
 781                         DroolsPdp lowestPriorityDifferentSite = null;
 782                         for(DroolsPdp pdp : listOfDesignated){
 783                                 // We need to determine if another PDP is the lowest priority
 784                                 if(nullSafeEquals(pdp.getSiteName(),mostRecentPrimary.getSiteName())){
 785                                         if(lowestPrioritySameSite == null){
 786                                                 if(lowestPriorityDifferentSite != null){
 787                                                         rejectedPdp = lowestPriorityDifferentSite;
 788                                                 }
 789                                                 lowestPrioritySameSite = pdp;
 790                                         }else{
 791                                                 if(pdp.getPdpId().equals((lowestPrioritySameSite.getPdpId()))){
 792                                                         continue;//nothing to compare
 793                                                 }
 794                                                 if(pdp.comparePriority(lowestPrioritySameSite) <0){
 795                                                         if(logger.isDebugEnabled()){
 796                                                                 logger.debug
 797                                                                 ("\nDesignatedWaiter.run: myPdp {}  listOfDesignated pdp ID: {}"
 798                                                                         + " has lower priority than pdp ID: {}",myPdp.getPdpId(), pdp.getPdpId(),
 799                                                                         lowestPrioritySameSite.getPdpId());
 800                                                         }
 801                                                         //we need to reject lowestPrioritySameSite
 802                                                         rejectedPdp = lowestPrioritySameSite;
 803                                                         lowestPrioritySameSite = pdp;
 804                                                 } else{
 805                                                         //we need to reject pdp and keep lowestPrioritySameSite
 806                                                         if(logger.isDebugEnabled()){
 807                                                                 logger.debug
 808                                                                 ("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {} "
 809                                                                         + " has higher priority than pdp ID: {}", myPdp.getPdpId(),pdp.getPdpId(),
 810                                                                         lowestPrioritySameSite.getPdpId());
 811                                                         }
 812                                                         rejectedPdp = pdp;
 813                                                 }
 814                                         }
 815                                 } else{
 816                                         if(lowestPrioritySameSite != null){
 817                                                 //if we already have a candidate for same site, we don't want to bother with different sites
 818                                                 rejectedPdp = pdp;
 819                                         } else{
 820                                                 if(lowestPriorityDifferentSite == null){
 821                                                         lowestPriorityDifferentSite = pdp;
 822                                                         continue;
 823                                                 }
 824                                                 if(pdp.getPdpId().equals((lowestPriorityDifferentSite.getPdpId()))){
 825                                                         continue;//nothing to compare
 826                                                 }
 827                                                 if(pdp.comparePriority(lowestPriorityDifferentSite) <0){
 828                                                         if(logger.isDebugEnabled()){
 829                                                                 logger.debug
 830                                                                 ("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 831                                                                         + " has lower priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 832                                                                         lowestPriorityDifferentSite.getPdpId());
 833                                                         }
 834                                                         //we need to reject lowestPriorityDifferentSite
 835                                                         rejectedPdp = lowestPriorityDifferentSite;
 836                                                         lowestPriorityDifferentSite = pdp;
 837                                                 } else{
 838                                                         //we need to reject pdp and keep lowestPriorityDifferentSite
 839                                                         if(logger.isDebugEnabled()){
 840                                                                 logger.debug
 841                                                                 ("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 842                                                                         + " has higher priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 843                                                                         lowestPriorityDifferentSite.getPdpId());
 844                                                         }
 845                                                         rejectedPdp = pdp;
 846                                                 }
 847                                         }
 848                                 }
 849                                 // If the rejectedPdp is myPdp, we need to stand it down and demote it.  Each pdp is responsible
 850                                 // for demoting itself
 851                                 if(rejectedPdp != null && nullSafeEquals(rejectedPdp.getPdpId(),myPdp.getPdpId())){
 852                                         if(logger.isDebugEnabled()){
 853                                                 logger.debug
 854                                                 ("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated myPdp ID: {}"
 855                                                         + " is NOT the lowest priority.  Executing stateManagement.demote()\n\n", myPdp.getPdpId(),
 856                                                          myPdp.getPdpId());
 857                                         }
 858                                         // We found that myPdp is on the listOfDesignated and it is not the lowest priority
 859                                         // So, we must demote it
 860                                         try {
 861                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 862                                                 myPdp.setDesignated(false);
 863                                                 pdpsConnector.setDesignated(myPdp, false);
 864                                                 isDesignated = false;
 865                                                 String standbyStatus = stateManagementFeature.getStandbyStatus();
 866                                                 if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 867                                                                 standbyStatus.equals(StateManagement.COLD_STANDBY))){
 868                                                         /*
 869                                                          * Only call demote if it is not already in the right state.  Don't worry about
 870                                                          * synching the lower level topic endpoint states.  That is done by the
 871                                                          * refreshStateAudit.
 872                                                          */
 873                                                         stateManagementFeature.demote();
 874                                                 }
 875                                         } catch (Exception e) {
 876                                                 myPdp.setDesignated(false);
 877                                                 pdpsConnector.setDesignated(myPdp, false);
 878                                                 isDesignated = false;
 879                                                 logger.error
 880                                                 ("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 881                                                                 + "demote myPdp {} myPdp.getPdpId(), message= {}", myPdp.getPdpId(),
 882                                                                 e);
 883                                         }
 884                                 }
 885                         } //end: for(DroolsPdp pdp : listOfDesignated)
 886                         if(lowestPrioritySameSite != null){
 887                                 lowestPriorityPdp = lowestPrioritySameSite;
 888                         } else {
 889                                 lowestPriorityPdp = lowestPriorityDifferentSite;
 890                         }
 891                         //now we have a valid value for lowestPriorityPdp
 892                         if(logger.isDebugEnabled()){
 893                                 logger.debug
 894                                 ("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 895                                         + "found the LOWEST priority pdp ID: {} "
 896                                         + " It is now the designatedPpd from the perspective of myPdp ID: {} \n\n",
 897                                          myPdp.getPdpId(), lowestPriorityPdp.getPdpId(), myPdp);
 898                         }
 899                         designatedPdp = lowestPriorityPdp;
 900
 901                 } else if(listOfDesignated.isEmpty()){
 902                         if(logger.isDebugEnabled()){
 903                                 logger.debug
 904                                 ("\nDesignatedWaiter.run: myPdp: {} listOfDesignated is: EMPTY.", myPdp.getPdpId());
 905                         }
 906                         designatedPdp = null;
 907                 } else{ //only one in listOfDesignated
 908                         if(logger.isDebugEnabled()){
 909                                 logger.debug
 910                                 ("\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 911                                         + "has ONE entry. PDP ID: {}", myPdp.getPdpId(), listOfDesignated.get(0).getPdpId());
 912                         }
 913                         designatedPdp = listOfDesignated.get(0);
 914                 }
 915                 return designatedPdp;
 916
 917         }
 918
 919         private class TimerUpdateClass extends TimerTask{
 920
 921                 @Override
 922                 public void run() {
 923                         try{
 924                                 if(logger.isDebugEnabled()){
 925                                         logger.debug("TimerUpdateClass.run: entry");
 926                                 }
 927                                 checkWaitTimer();
 928                         }catch(Exception e){
 929                                 logger.error("TimerUpdateClass.run caught an unexpected exception: ", e);
 930                         }
 931                         if(logger.isDebugEnabled()){
 932                                 logger.debug("TimerUpdateClass.run.exit");
 933                         }
 934                 }
 935         }
 936         @Override
 937         public void checkThreadStatus() {
 938                 checkWaitTimer();
 939         }
 940
 941         private void checkWaitTimer(){
 942                 synchronized(checkWaitTimerLock){
 943                         try{
 944                                 if(logger.isDebugEnabled()){
 945                                         logger.debug("checkWaitTimer: entry");
 946                                 }
 947                                 Date now = new Date();
 948                                 long nowMs = now.getTime();
 949                                 long waitTimerMs = waitTimerLastRunDate.getTime();
 950
 951                                 //give it 10 times leeway
 952                                 if((nowMs - waitTimerMs)  > 10*pdpUpdateInterval){
 953                                         if(allSeemsWell==null || allSeemsWell){
 954                                                 allSeemsWell = false;
 955                                                 if(logger.isDebugEnabled()){
 956                                                         logger.debug("checkWaitTimer: calling allSeemsWell with ALLNOTWELL param");
 957                                                 }
 958                                                 stateManagementFeature.allSeemsWell(this.getClass().getName(),
 959                                                                 StateManagementFeatureAPI.ALLNOTWELL_STATE,
 960                                                                 "DesignationWaiter/ElectionHandler has STALLED");
 961                                         }
 962                                         logger.error("checkWaitTimer: nowMs - waitTimerMs = {}"
 963                                                         + ", exceeds 10* pdpUpdateInterval = {}"
 964                                                         + " DesignationWaiter is STALLED!", (nowMs - waitTimerMs), (10*pdpUpdateInterval));
 965                                 }else if(allSeemsWell==null || !allSeemsWell){
 966                                         allSeemsWell = true;
 967                                         stateManagementFeature.allSeemsWell(this.getClass().getName(),
 968                                                         StateManagementFeatureAPI.ALLSEEMSWELL_STATE,
 969                                                         "DesignationWaiter/ElectionHandler has RESUMED");
 970                                         logger.info("DesignationWaiter/ElectionHandler has RESUMED");
 971                                 }
 972                                 if(logger.isDebugEnabled()){
 973                                         logger.debug("checkWaitTimer: exit");
 974                                 }
 975                         }catch(Exception e){
 976                                 logger.error("checkWaitTimer: caught unexpected exception: ", e);
 977                         }
 978                 }
 979         }
 980
 981         private long getDWaiterStartMs(){
 982                 Date now = new Date();
 983
 984                 // Retrieve the ms since the epoch
 985                 long nowMs = now.getTime();
 986
 987                 // Time since the end of the last pdpUpdateInterval multiple
 988                 long nowModMs = nowMs % pdpUpdateInterval;
 989
 990                 // Time to the start of the next pdpUpdateInterval multiple
 991                 long startMs = 2*pdpUpdateInterval - nowModMs;
 992
 993                 // Give the start time a minimum of a 5 second cushion
 994                 if(startMs < 5000){
 995                         // Start at the beginning  of following interval
 996                         startMs = pdpUpdateInterval + startMs;
 997                 }
 998                 return startMs;
 999         }
1000
1001         private boolean nullSafeEquals(Object one, Object two){
1002                 if(one == null && two == null){
1003                         return true;
1004                 }
1005                 if(one != null && two != null){
1006                         return one.equals(two);
1007                 }
1008                 return false;
1009         }
1010
1011         public String getPdpdNowActive(){
1012                 return pdpdNowActive;
1013         }
1014
1015         public String getPdpdLastActive(){
1016                 return pdpdLastActive;
1017         }
1018 }