gerrit.onap Code Review - policy/drools-pdp.git/blob

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * feature-active-standby-management
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.policy.drools.activestandby;
  22
  23 import java.util.ArrayList;
  24 import java.util.Collection;
  25 import java.util.Date;
  26 import java.util.Timer;
  27 import java.util.TimerTask;
  28
  29 import org.onap.policy.common.im.StateManagement;
  30 import org.slf4j.Logger;
  31 import org.slf4j.LoggerFactory;
  32 import org.onap.policy.drools.statemanagement.StateManagementFeatureAPI;
  33
  34 public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
  35         // get an instance of logger
  36         private final static Logger  logger = LoggerFactory.getLogger(DroolsPdpsElectionHandler.class);
  37         private DroolsPdpsConnector pdpsConnector;
  38         private Object checkWaitTimerLock = new Object();
  39         private Object designationWaiterLock = new Object();
  40
  41         /*
  42          * Must be static, so it can be referenced by JpaDroolsPdpsConnector,
  43          * without requiring a reference to the election handler instantiation.
  44          */
  45         private static DroolsPdp myPdp;
  46
  47         private DesignationWaiter designationWaiter;
  48         private Timer updateWorker;
  49         private Timer waitTimer;
  50         private Date waitTimerLastRunDate;
  51
  52         // The interval between checks of the DesignationWaiter to be sure it is running.
  53         private int pdpCheckInterval;
  54
  55         // The interval between runs of the DesignationWaiter
  56         private int pdpUpdateInterval;
  57
  58         private volatile boolean isDesignated;
  59
  60         private String pdpdNowActive;
  61         private String pdpdLastActive;
  62
  63         /*
  64          * Start allSeemsWell with a value of null so that, on the first run
  65          * of the checkWaitTimer it will set the value in IntegrityMonitor
  66          * regardless of whether it needs to be set to true or false.
  67          */
  68         private Boolean allSeemsWell=null;
  69
  70         private StateManagementFeatureAPI stateManagementFeature;
  71
  72         private static boolean isUnitTesting = false;
  73         public static void setIsUnitTesting(boolean val){
  74                 isUnitTesting = val;
  75         }
  76         private static boolean isStalled = false;
  77         public static void setIsStalled(boolean val){
  78                 isStalled = val;
  79         }
  80
  81         public DroolsPdpsElectionHandler(DroolsPdpsConnector pdps, DroolsPdp myPdp){
  82                 if (pdps == null) {
  83                         logger.error("DroolsPdpsElectinHandler(): pdpsConnector==null");
  84                         throw new IllegalArgumentException("DroolsPdpsElectinHandler(): pdpsConnector==null");
  85                 }
  86                 if (myPdp == null){
  87                         logger.error("DroolsPdpsElectinHandler(): droolsPdp==null");
  88                         throw new IllegalArgumentException("DroolsPdpsElectinHandler(): DroolsPdp==null");
  89                 }
  90
  91                 pdpdNowActive = null;
  92                 pdpdLastActive = null;
  93                 this.pdpsConnector = pdps;
  94                 DroolsPdpsElectionHandler.myPdp = myPdp;
  95                 this.isDesignated = false;
  96                 pdpCheckInterval = 3000;
  97                 try{
  98                         pdpCheckInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(ActiveStandbyProperties.PDP_CHECK_INVERVAL));
  99                 }catch(Exception e){
 100                         logger.error
 101                         ("Could not get pdpCheckInterval property. Using default {}",pdpCheckInterval, e);
 102                 }
 103                 pdpUpdateInterval = 2000;
 104                 try{
 105                         pdpUpdateInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(ActiveStandbyProperties.PDP_UPDATE_INTERVAL));
 106                 }catch(Exception e){
 107                         logger.error
 108                         ("Could not get pdpUpdateInterval property. Using default {} ", pdpUpdateInterval, e);
 109                 }
 110
 111                 Date now = new Date();
 112
 113                 // Retrieve the ms since the epoch
 114                 long nowMs = now.getTime();
 115
 116                 // Create the timer which will update the updateDate in DroolsPdpEntity table.
 117                 // This is the heartbeat
 118                 updateWorker = new Timer();
 119
 120                 // Schedule the TimerUpdateClass to run at 100 ms and run at pdpCheckInterval ms thereafter
 121                 // NOTE: The first run of the TimerUpdateClass results in myPdp being added to the
 122                 // drools droolsPdpEntity table.
 123                 updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
 124
 125                 // Create the timer which will run the election algorithm
 126                 waitTimer = new Timer();
 127
 128                 // Schedule it to start in startMs ms (so it will run after the updateWorker and run at pdpUpdateInterval ms thereafter
 129                 long startMs = getDWaiterStartMs();
 130                 designationWaiter = new DesignationWaiter();
 131                 waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
 132                 waitTimerLastRunDate = new Date(nowMs + startMs);
 133
 134                 //Get the StateManagementFeature instance
 135
 136                 for (StateManagementFeatureAPI feature : StateManagementFeatureAPI.impl.getList())
 137                 {
 138                         if (feature.getResourceName().equals(myPdp.getPdpId()))
 139                         {
 140                                 if(logger.isDebugEnabled()){
 141                                         logger.debug("DroolsPdpsElectionHandler: Found StateManagementFeature"
 142                                                 + " with resourceName: {}", myPdp.getPdpId());
 143                                 }
 144                                 stateManagementFeature = feature;
 145                                 break;
 146                         }
 147                 }
 148                 if(stateManagementFeature == null){
 149                         logger.error("DroolsPdpsElectionHandler failed to initialize.  "
 150                                         + "Unable to get instance of StateManagementFeatureAPI "
 151                                         + "with resourceID: {}", myPdp.getPdpId());
 152                 }
 153         }
 154
 155         /*
 156          * When the JpaDroolsPdpsConnector.standDown() method is invoked, it needs
 157          * access to myPdp, so it can keep its designation status in sync with the
 158          * DB.
 159          */
 160         public static void setMyPdpDesignated(boolean designated) {
 161                 if(logger.isDebugEnabled()){
 162                         logger.debug
 163                         ("setMyPdpDesignated: designated= {}", designated);
 164                 }
 165                 myPdp.setDesignated(designated);
 166         }
 167
 168         private class DesignationWaiter extends TimerTask {
 169                 // get an instance of logger
 170                 private final Logger  logger = LoggerFactory.getLogger(DesignationWaiter.class);
 171
 172                 @Override
 173                 public void run() {
 174                         try{
 175                                 if(logger.isDebugEnabled()){
 176                                         logger.debug
 177                                         ("DesignatedWaiter.run: Entering");
 178                                 }
 179
 180                                 //This is for testing the checkWaitTimer
 181                                 if(isUnitTesting && isStalled){
 182                                         if(logger.isDebugEnabled()){
 183                                                 logger.debug("DesignatedWaiter.run: isUnitTesting = {} isStalled = {}", isUnitTesting, isStalled);
 184                                         }
 185                                         return;
 186                                 }
 187
 188                                 synchronized (designationWaiterLock) {
 189
 190                                         if(logger.isDebugEnabled()){
 191                                                 logger.debug
 192                                                 ("DesignatedWaiter.run: Entering synchronized block");
 193                                         }
 194
 195                                         //It is possible that multiple PDPs are designated lead.  So, we will make a list of all designated
 196                                         //PDPs and then decide which one really should be designated at the end.
 197                                         ArrayList<DroolsPdp> listOfDesignated = new ArrayList<DroolsPdp>();
 198
 199                                         Collection<DroolsPdp> pdps = pdpsConnector.getDroolsPdps();
 200                                         DroolsPdp designatedPdp = null;
 201
 202                                         if(logger.isDebugEnabled()){
 203                                                 logger.debug
 204                                                 ("DesignatedWaiter.run: pdps.size= {}", pdps.size());
 205                                         }
 206
 207                                         //This is only true if all designated PDPs have failed
 208                                         boolean designatedPdpHasFailed = pdpsConnector.hasDesignatedPdpFailed(pdps);
 209                                         if(logger.isDebugEnabled()){
 210                                                 logger.debug
 211                                                 ("DesignatedWaiter.run: designatedPdpHasFailed= {}", designatedPdpHasFailed);
 212                                         }
 213                                         for (DroolsPdp pdp : pdps) {
 214                                                 if(logger.isDebugEnabled()){
 215                                                         logger.debug
 216                                                         ("DesignatedWaiter.run: evaluating pdp ID: {}", pdp.getPdpId());
 217                                                 }
 218
 219                                                 /*
 220                                                  * Note: side effect of isPdpCurrent is that any stale but
 221                                                  * designated PDPs will be marked as un-designated.
 222                                                  */
 223                                                 boolean isCurrent = pdpsConnector.isPdpCurrent(pdp);
 224
 225                                                 /*
 226                                                  * We can't use stateManagement.getStandbyStatus() here, because
 227                                                  * we need the standbyStatus, not for this PDP, but for the PDP
 228                                                  * being processed by this loop iteration.
 229                                                  */
 230                                                 String standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 231                                                 if(standbyStatus==null){
 232                                                         // Treat this case as a cold standby -- if we
 233                                                         // abort here, no sessions will be created in a
 234                                                         // single-node test environment.
 235                                                         standbyStatus = StateManagement.COLD_STANDBY;
 236                                                 }
 237                                                 if(logger.isDebugEnabled()){
 238                                                         logger.debug
 239                                                         ("DesignatedWaiter.run: PDP= {},  isCurrent= {}", pdp.getPdpId(), isCurrent);
 240                                                 }
 241
 242                                                 /*
 243                                                  * There are 4 combinations of isDesignated and isCurrent.  We will examine each one in-turn
 244                                                  * and evaluate the each pdp in the list of pdps against each combination.
 245                                                  *
 246                                                  * This is the first combination of isDesignated and isCurrent
 247                                                  */
 248                                                 if (pdp.isDesignated()  &&  isCurrent) {
 249                                                         //It is current, but it could have a standbystatus=coldstandby / hotstandby
 250                                                         //If so, we need to stand it down and demote it
 251                                                         if(!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)){
 252                                                                 if(pdp.getPdpId().equals(myPdp.getPdpId())){
 253                                                                         if(logger.isDebugEnabled()){
 254                                                                                 logger.debug
 255                                                                                 ("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 256                                                                                         + "butstandbystatus is not providingservice. "
 257                                                                                         + " Executing stateManagement.demote()" + "\n\n", myPdp.getPdpId());
 258                                                                         }
 259                                                                         // So, we must demote it
 260                                                                         try {
 261                                                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 262                                                                                 //This will change isDesignated and it can enter another if(combination) below
 263                                                                                 pdpsConnector.standDownPdp(pdp.getPdpId());
 264                                                                                 myPdp.setDesignated(false);
 265                                                                                 isDesignated = false;
 266                                                                                 if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 267                                                                                                 standbyStatus.equals(StateManagement.COLD_STANDBY))){
 268                                                                                         /*
 269                                                                                          * Only demote it if it appears it has not already been demoted. Don't worry
 270                                                                                          * about synching with the topic endpoint states.  That is done by the
 271                                                                                          * refreshStateAudit
 272                                                                                          */
 273                                                                                         stateManagementFeature.demote();
 274                                                                                 }
 275                                                                                 //update the standbystatus to check in a later combination of isDesignated and isCurrent
 276                                                                                 standbyStatus=stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 277                                                                         } catch (Exception e) {
 278                                                                                 logger.error
 279                                                                                 ("DesignatedWaiter.run: myPdp: {} "
 280                                                                                                 + "Caught Exception attempting to demote myPdp,"
 281                                                                                                 + "message= {}", myPdp.getPdpId(), e);
 282                                                                         }
 283                                                                 }else{
 284                                                                         // Don't demote a remote PDP that is current.  It should catch itself
 285                                                                         if(logger.isDebugEnabled()){
 286                                                                                 logger.debug
 287                                                                                 ("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 288                                                                                         + "but standbystatus is not providingservice. "
 289                                                                                         + " Cannot execute stateManagement.demote() since it it is not myPdp\n\n", myPdp.getPdpId());
 290                                                                         }
 291                                                                 }
 292
 293                                                         }else{
 294                                                                 // If we get here, it is ok to be on the list
 295                                                                 if(logger.isDebugEnabled()){
 296                                                                         logger.debug
 297                                                                         ("DesignatedWaiter.run: PDP= {} is designated, current and {} Noting PDP as "
 298                                                                                 + "designated, standbyStatus= {}", pdp.getPdpId(), standbyStatus, standbyStatus);
 299                                                                 }
 300                                                                 listOfDesignated.add(pdp);
 301                                                         }
 302
 303
 304                                                 }
 305
 306
 307                                                 /*
 308                                                  * The second combination of isDesignated and isCurrent
 309                                                  *
 310                                                  * PDP is designated but not current; it has failed.   So we stand it down (it doesn't matter what
 311                                                  * its standbyStatus is). None of these go on the list.
 312                                                  */
 313                                                 if (pdp.isDesignated()  &&  !isCurrent) {
 314                                                         if(logger.isDebugEnabled()){
 315                                                                 logger.debug
 316                                                                 ("INFO: DesignatedWaiter.run: PDP= {} is currently designated but is not current; "
 317                                                                         + "it has failed.  Standing down.  standbyStatus= {}", pdp.getPdpId(), standbyStatus);
 318                                                         }
 319                                                         /*
 320                                                          * Changes designated to 0 but it is still potentially providing service
 321                                                          * Will affect isDesignated, so, it can enter an if(combination) below
 322                                                          */
 323                                                         pdpsConnector.standDownPdp(pdp.getPdpId());
 324
 325                                                         //need to change standbystatus to coldstandby
 326                                                         if (pdp.getPdpId().equals(myPdp.getPdpId())){
 327                                                                 if(logger.isDebugEnabled()){
 328                                                                         logger.debug
 329                                                                         ("\n\nDesignatedWaiter.run: myPdp {} is not Current. "
 330                                                                                 + " Executing stateManagement.disableFailed()\n\n", myPdp.getPdpId());
 331                                                                 }
 332                                                                 // We found that myPdp is designated but not current
 333                                                                 // So, we must cause it to disableFail
 334                                                                 try {
 335                                                                         myPdp.setDesignated(false);
 336                                                                         pdpsConnector.setDesignated(myPdp, false);
 337                                                                         isDesignated = false;
 338                                                                         stateManagementFeature.disableFailed();
 339                                                                 } catch (Exception e) {
 340                                                                         logger.error
 341                                                                         ("DesignatedWaiter.run: myPdp: {} Caught Exception "
 342                                                                                         + "attempting to disableFail myPdp {}, message= {}",
 343                                                                                         myPdp.getPdpId(), myPdp.getPdpId(), e);
 344                                                                 }
 345                                                         } else { //it is a remote PDP that is failed
 346                                                                 if(logger.isDebugEnabled()){
 347                                                                         logger.debug
 348                                                                         ("\n\nDesignatedWaiter.run: PDP {} is not Current. "
 349                                                                                 + " Executing stateManagement.disableFailed(otherResourceName)\n\n", pdp.getPdpId() );
 350                                                                 }
 351                                                                 // We found a PDP is designated but not current
 352                                                                 // We already called standdown(pdp) which will change designated to false
 353                                                                 // Now we need to disableFail it to get its states in synch.  The standbyStatus
 354                                                                 // should equal coldstandby
 355                                                                 try {
 356                                                                         stateManagementFeature.disableFailed(pdp.getPdpId());
 357                                                                 } catch (Exception e) {
 358                                                                         logger.error
 359                                                                         ("DesignatedWaiter.run: for PDP {}  Caught Exception attempting to "
 360                                                                                         + "disableFail({}), message= {}",
 361                                                                                         pdp.getPdpId(), pdp.getPdpId(), e);
 362                                                                 }
 363
 364                                                         }
 365                                                         continue; //we are not going to do anything else with this pdp
 366                                                 }
 367
 368                                                 /*
 369                                                  * The third combination of isDesignated and isCurrent
 370                                                  * /*
 371                                                  * If a PDP is not currently designated but is providing service (erroneous, but recoverable) or hot standby
 372                                                  * we can add it to the list of possible designated if all the designated have failed
 373                                                  */
 374                                                 if (!pdp.isDesignated() && isCurrent){
 375                                                         if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 376                                                                         standbyStatus.equals(StateManagement.COLD_STANDBY))){
 377                                                                 if(logger.isDebugEnabled()){
 378                                                                         logger.debug("\n\nDesignatedWaiter.run: PDP {}"
 379                                                                                 + " is NOT designated but IS current and"
 380                                                                                 + " has a standbystatus= {}", pdp.getPdpId(), standbyStatus);
 381                                                                 }
 382                                                                 // Since it is current, we assume it can adjust its own state.
 383                                                                 // We will demote if it is myPdp
 384                                                                 if(pdp.getPdpId().equals(myPdp.getPdpId())){
 385                                                                         //demote it
 386                                                                         if(logger.isDebugEnabled()){
 387                                                                                 logger.debug("DesignatedWaiter.run: PDP {} going to "
 388                                                                                         + "setDesignated = false and calling stateManagement.demote",  pdp.getPdpId());
 389                                                                         }
 390                                                                         try {
 391                                                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 392                                                                                 pdpsConnector.setDesignated(myPdp, false);
 393                                                                                 myPdp.setDesignated(false);
 394                                                                                 isDesignated = false;
 395                                                                                 //This is definitely not a redundant call.  It is attempting to correct a problem
 396                                                                                 stateManagementFeature.demote();
 397                                                                                 //recheck the standbystatus
 398                                                                                 standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 399                                                                         } catch (Exception e) {
 400                                                                                 logger.error
 401                                                                                 ("DesignatedWaiter.run: myPdp: {} Caught Exception "
 402                                                                                                 + "attempting to demote myPdp {}, message = {}",  myPdp.getPdpId(),
 403                                                                                                 myPdp.getPdpId(), e);
 404                                                                         }
 405
 406                                                                 }
 407                                                         }
 408                                                         if(standbyStatus.equals(StateManagement.HOT_STANDBY) && designatedPdpHasFailed){
 409                                                                 //add it to the list
 410                                                                 if(logger.isDebugEnabled()){
 411                                                                         logger.debug
 412                                                                         ("INFO: DesignatedWaiter.run: PDP= {}"
 413                                                                                 + " is not designated but is {} and designated PDP "
 414                                                                                 + "has failed.  standbyStatus= {}", pdp.getPdpId(),
 415                                                                                 standbyStatus, standbyStatus);
 416                                                                 }
 417                                                                 listOfDesignated.add(pdp);
 418                                                         }
 419                                                         continue; //done with this one
 420                                                 }
 421
 422                                                 /*
 423                                                  * The fourth combination of isDesignated and isCurrent
 424                                                  *
 425                                                  * We are not going to put any of these on the list since it appears they have failed.
 426
 427                                                  *
 428                                                  */
 429                                                 if(!pdp.isDesignated() && !isCurrent) {
 430                                                         if(logger.isDebugEnabled()){
 431                                                                 logger.debug
 432                                                                 ("INFO: DesignatedWaiter.run: PDP= {} "
 433                                                                         + "designated= {}, current= {}, "
 434                                                                         + "designatedPdpHasFailed= {}, "
 435                                                                         + "standbyStatus= {}",pdp.getPdpId(),
 436                                                                         pdp.isDesignated(), isCurrent, designatedPdpHasFailed, standbyStatus);
 437                                                         }
 438                                                         if(!standbyStatus.equals(StateManagement.COLD_STANDBY)){
 439                                                                 //stand it down
 440                                                                 //disableFail it
 441                                                                 pdpsConnector.standDownPdp(pdp.getPdpId());
 442                                                                 if(pdp.getPdpId().equals(myPdp.getPdpId())){
 443                                                                         /*
 444                                                                          * I don't actually know how this condition could happen, but if it did, we would want
 445                                                                          * to declare it failed.
 446                                                                          */
 447                                                                         if(logger.isDebugEnabled()){
 448                                                                                 logger.debug
 449                                                                                 ("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 450                                                                                         + " Executing stateManagement.disableFailed()\n\n", myPdp.getPdpId());
 451                                                                         }
 452                                                                         // So, we must disableFail it
 453                                                                         try {
 454                                                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 455                                                                                 pdpsConnector.setDesignated(myPdp, false);
 456                                                                                 myPdp.setDesignated(false);
 457                                                                                 isDesignated = false;
 458                                                                                 stateManagementFeature.disableFailed();
 459                                                                         } catch (Exception e) {
 460                                                                                 logger.error
 461                                                                                 ("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 462                                                                                                 + "disableFail myPdp {}, message= {}",
 463                                                                                                  myPdp.getPdpId(), myPdp.getPdpId(), e);
 464                                                                         }
 465                                                                 }else{//it is remote
 466                                                                         if(logger.isDebugEnabled()){
 467                                                                                 logger.debug
 468                                                                                 ("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 469                                                                                         + " Executing stateManagement.disableFailed({})\n\n",
 470                                                                                         myPdp.getPdpId(), pdp.getPdpId());
 471                                                                         }
 472                                                                         // We already called standdown(pdp) which will change designated to false
 473                                                                         // Now we need to disableFail it to get its states in sync.  StandbyStatus = coldstandby
 474                                                                         try {
 475                                                                                 stateManagementFeature.disableFailed(pdp.getPdpId());
 476                                                                         } catch (Exception e) {
 477                                                                                 logger.error
 478                                                                                 ("DesignatedWaiter.run: for PDP {}"
 479                                                                                                 + " Caught Exception attempting to disableFail({})"
 480                                                                                                 + ", message=", pdp.getPdpId(), pdp.getPdpId(), e);
 481                                                                         }
 482                                                                 }
 483                                                         }
 484                                                 }
 485
 486
 487                                         } // end pdps loop
 488
 489                                         /*
 490                                          * We have checked the four combinations of isDesignated and isCurrent.  Where appropriate,
 491                                          * we added the PDPs to the potential list of designated pdps
 492                                          *
 493                                          * We need to give priority to pdps on the same site that is currently being used
 494                                          * First, however, we must sanitize the list of designated to make sure their are
 495                                          * only designated members or non-designated members.  There should not be both in
 496                                          * the list. Because there are real time delays, it is possible that both types could
 497                                          * be on the list.
 498                                          */
 499
 500                                         listOfDesignated = santizeDesignatedList(listOfDesignated);
 501
 502                                         /*
 503                                          * We need to figure out the last pdp that was the primary so we can get the last site
 504                                          * name and the last session numbers.  We need to create a "dummy" droolspdp since
 505                                          * it will be used in later comparisons and cannot be null.
 506                                          */
 507
 508                                         DroolsPdp mostRecentPrimary = computeMostRecentPrimary(pdps, listOfDesignated);
 509
 510                                         if(mostRecentPrimary != null){
 511                                                 pdpdLastActive = mostRecentPrimary.getPdpId();
 512                                         }
 513
 514
 515                                         /*
 516                                          * It is possible to get here with more than one pdp designated and providingservice. This normally
 517                                          * occurs when there is a race condition with multiple nodes coming up at the same time. If that is
 518                                          * the case we must determine which one is the one that should be designated and which one should
 519                                          * be demoted.
 520                                          *
 521                                          * It is possible to have 0, 1, 2 or more but not all, or all designated.
 522                                          *   If we have one designated and current, we chose it and are done
 523                                          *   If we have 2 or more, but not all, we must determine which one is in the same site as
 524                                          *   the previously designated pdp.
 525                                          */
 526
 527                                         designatedPdp = computeDesignatedPdp(listOfDesignated, mostRecentPrimary);
 528                                         if(designatedPdp != null){
 529                                                 pdpdNowActive = designatedPdp.getPdpId();
 530                                         }
 531
 532                                         if (designatedPdp == null) {
 533                                                 logger.warn
 534                                                 ("WARNING: DesignatedWaiter.run: No viable PDP found to be Designated. designatedPdp still null.");
 535                                                 // Just to be sure the parameters are correctly set
 536                                                 myPdp.setDesignated(false);
 537                                                 pdpsConnector.setDesignated(myPdp,false);
 538                                                 isDesignated = false;
 539
 540                                                 waitTimerLastRunDate = new Date();
 541                                                 if(logger.isDebugEnabled()){
 542                                                         logger.debug("DesignatedWaiter.run (designatedPdp == null) waitTimerLastRunDate = {}", waitTimerLastRunDate);
 543                                                 }
 544                                                 myPdp.setUpdatedDate(waitTimerLastRunDate);
 545                                                 pdpsConnector.update(myPdp);
 546
 547                                                 return;
 548
 549                                         } else if (designatedPdp.getPdpId().equals(myPdp.getPdpId())) {
 550                                                 if(logger.isDebugEnabled()){
 551                                                         logger.debug
 552                                                         ("DesignatedWaiter.run: designatedPdp is PDP={}", myPdp.getPdpId());
 553                                                 }
 554                                                 /*
 555                                                  * update function expects myPdp.isDesignated to be true.
 556                                                  */
 557                                                 try {
 558                                                         //Keep the order like this.  StateManagement is last since it triggers controller init
 559                                                         myPdp.setDesignated(true);
 560                                                         myPdp.setDesignatedDate(new Date());
 561                                                         pdpsConnector.setDesignated(myPdp, true);
 562                                                         isDesignated = true;
 563                                                         String standbyStatus = stateManagementFeature.getStandbyStatus();
 564                                                         if(!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)){
 565                                                                 /*
 566                                                                  * Only call promote if it is not already in the right state.  Don't worry about
 567                                                                  * synching the lower level topic endpoint states.  That is done by the
 568                                                                  * refreshStateAudit.
 569                                                                  * Note that we need to fetch the session list from 'mostRecentPrimary'
 570                                                                  * at this point -- soon, 'mostRecentPrimary' will be set to this host.
 571                                                                  */
 572                                                                 //this.sessions = mostRecentPrimary.getSessions();
 573                                                                 stateManagementFeature.promote();
 574                                                         }
 575                                                 } catch (Exception e) {
 576                                                         logger.error
 577                                                         ("ERROR: DesignatedWaiter.run: Caught Exception attempting to promote PDP={}"
 578                                                                         + ", message=", myPdp.getPdpId(), e);
 579                                                         myPdp.setDesignated(false);
 580                                                         pdpsConnector.setDesignated(myPdp,false);
 581                                                         isDesignated = false;
 582                                                         //If you can't promote it, demote it
 583                                                         try {
 584                                                                 String standbyStatus = stateManagementFeature.getStandbyStatus();
 585                                                                 if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 586                                                                                 standbyStatus.equals(StateManagement.COLD_STANDBY))){
 587                                                                         /*
 588                                                                          * Only call demote if it is not already in the right state.  Don't worry about
 589                                                                          * synching the lower level topic endpoint states.  That is done by the
 590                                                                          * refreshStateAudit.
 591                                                                          */
 592                                                                         stateManagementFeature.demote();
 593                                                                 }
 594                                                         } catch (Exception e1) {
 595                                                                 logger.error
 596                                                                 ("ERROR: DesignatedWaiter.run: Caught StandbyStatusException "
 597                                                                                 + "attempting to promote then demote PDP={}, message=",
 598                                                                                 myPdp.getPdpId(), e1);
 599                                                         }
 600
 601                                                 }
 602                                                 waitTimerLastRunDate = new Date();
 603                                                 if(logger.isDebugEnabled()){
 604                                                         logger.debug("DesignatedWaiter.run (designatedPdp.getPdpId().equals(myPdp.getPdpId())) "
 605                                                                 + "waitTimerLastRunDate = " + waitTimerLastRunDate);
 606                                                 }
 607                                                 myPdp.setUpdatedDate(waitTimerLastRunDate);
 608                                                 pdpsConnector.update(myPdp);
 609
 610                                                 return;
 611                                         }
 612                                         isDesignated = false;
 613
 614                                 } // end synchronized
 615                                 if(logger.isDebugEnabled()){
 616                                         logger.debug
 617                                         ("DesignatedWaiter.run: myPdp: {}; Returning, isDesignated= {}",
 618                                                 isDesignated, myPdp.getPdpId());
 619                                 }
 620
 621                                 Date tmpDate = new Date();
 622                                 if(logger.isDebugEnabled()){
 623                                         logger.debug("DesignatedWaiter.run (end of run) waitTimerLastRunDate = {}", tmpDate);
 624                                 }
 625
 626                                 waitTimerLastRunDate = tmpDate;
 627                                 myPdp.setUpdatedDate(waitTimerLastRunDate);
 628                                 pdpsConnector.update(myPdp);
 629
 630                         }catch(Exception e){
 631                                 logger.error("DesignatedWaiter.run caught an unexpected exception: ", e);
 632                         }
 633                 } // end run
 634         }
 635
 636         public ArrayList<DroolsPdp> santizeDesignatedList(ArrayList<DroolsPdp> listOfDesignated){
 637
 638                 boolean containsDesignated = false;
 639                 boolean containsHotStandby = false;
 640                 ArrayList<DroolsPdp> listForRemoval = new ArrayList<DroolsPdp>();
 641                 for(DroolsPdp pdp : listOfDesignated){
 642                         if(logger.isDebugEnabled()){
 643                                 logger.debug
 644                                 ("DesignatedWaiter.run sanitizing: pdp = {}"
 645                                         + " isDesignated = {}",pdp.getPdpId(), pdp.isDesignated());
 646                         }
 647                         if(pdp.isDesignated()){
 648                                 containsDesignated = true;
 649                         }else {
 650                                 containsHotStandby = true;
 651                                 listForRemoval.add(pdp);
 652                         }
 653                 }
 654                 if(containsDesignated && containsHotStandby){
 655                         //remove the hot standby from the list
 656                         listOfDesignated.removeAll(listForRemoval);
 657                         containsHotStandby = false;
 658                 }
 659                 return listOfDesignated;
 660         }
 661
 662         public DroolsPdp computeMostRecentPrimary(Collection<DroolsPdp> pdps, ArrayList<DroolsPdp> listOfDesignated){
 663                 boolean containsDesignated = false;
 664                 for(DroolsPdp pdp : listOfDesignated){
 665                         if(pdp.isDesignated()){
 666                                 containsDesignated = true;
 667                         }
 668                 }
 669                 DroolsPdp mostRecentPrimary = new DroolsPdpImpl(null, true, 1, new Date(0));
 670                 mostRecentPrimary.setSiteName(null);
 671                 if(logger.isDebugEnabled()){
 672                         logger.debug
 673                         ("DesignatedWaiter.run listOfDesignated.size() = {}", listOfDesignated.size());
 674                 }
 675                 if(listOfDesignated.size() <=1){
 676                         if(logger.isDebugEnabled()){
 677                                 logger.debug("DesignatedWainter.run: listOfDesignated.size <=1");
 678                         }
 679                         //Only one or none is designated or hot standby.  Choose the latest designated date
 680                         for(DroolsPdp pdp : pdps){
 681                                 if(logger.isDebugEnabled()){
 682                                         logger.debug
 683                                         ("DesignatedWaiter.run pdp = {}"
 684                                                 + " pdp.getDesignatedDate() = {}", pdp.getPdpId(), pdp.getDesignatedDate());
 685                                 }
 686                                 if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 687                                         mostRecentPrimary = pdp;
 688                                         if(logger.isDebugEnabled()){
 689                                                 logger.debug
 690                                                 ("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 691                                         }
 692                                 }
 693                         }
 694                 }else if(listOfDesignated.size() == pdps.size()){
 695                         if(logger.isDebugEnabled()){
 696                                 logger.debug("DesignatedWainter.run: listOfDesignated.size = pdps.size() which is {}", pdps.size());
 697                         }
 698                         //They are all designated or all hot standby.
 699                         mostRecentPrimary = null;
 700                         for(DroolsPdp pdp : pdps){
 701                                 if(mostRecentPrimary == null){
 702                                         mostRecentPrimary = pdp;
 703                                         continue;
 704                                 }
 705                                 if(containsDesignated){ //Choose the site of the first designated date
 706                                         if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) < 0){
 707                                                 mostRecentPrimary = pdp;
 708                                                 if(logger.isDebugEnabled()){
 709                                                         logger.debug
 710                                                         ("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 711                                                 }
 712                                         }
 713                                 }else{ //Choose the site with the latest designated date
 714                                         if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 715                                                 mostRecentPrimary = pdp;
 716                                                 if(logger.isDebugEnabled()){
 717                                                         logger.debug
 718                                                         ("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 719                                                 }
 720                                         }
 721                                 }
 722                         }
 723                 }else{
 724                         if(logger.isDebugEnabled()){
 725                                 logger.debug("DesignatedWainter.run: Some but not all are designated or hot standby. ");
 726                         }
 727                         //Some but not all are designated or hot standby.
 728                         if(containsDesignated){
 729                                 if(logger.isDebugEnabled()){
 730                                         logger.debug("DesignatedWainter.run: containsDesignated = {}", containsDesignated);
 731                                 }
 732                                 /*
 733                                  * The list only contains designated.  This is a problem.  It is most likely a race
 734                                  * condition that resulted in two thinking they should be designated. Choose the
 735                                  * site with the latest designated date for the pdp not included on the designated list.
 736                                  * This should be the site that had the last designation before this race condition
 737                                  * occurred.
 738                                  */
 739                                 for(DroolsPdp pdp : pdps){
 740                                         if(listOfDesignated.contains(pdp)){
 741                                                 continue; //Don't consider this entry
 742                                         }
 743                                         if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 744                                                 mostRecentPrimary = pdp;
 745                                                 if(logger.isDebugEnabled()){
 746                                                         logger.debug
 747                                                         ("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 748                                                 }
 749                                         }
 750                                 }
 751                         }else{
 752                                 if(logger.isDebugEnabled()){
 753                                         logger.debug("DesignatedWainter.run: containsDesignated = {}", containsDesignated);
 754                                 }
 755                                 //The list only contains hot standby. Choose the site of the latest designated date
 756                                 for(DroolsPdp pdp : pdps){
 757                                         if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 758                                                 mostRecentPrimary = pdp;
 759                                                 if(logger.isDebugEnabled()){
 760                                                         logger.debug
 761                                                         ("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 762                                                 }
 763                                         }
 764                                 }
 765                         }
 766                 }
 767                 return mostRecentPrimary;
 768         }
 769
 770         public DroolsPdp computeDesignatedPdp(ArrayList<DroolsPdp> listOfDesignated, DroolsPdp mostRecentPrimary){
 771                 DroolsPdp designatedPdp = null;
 772                 DroolsPdp lowestPriorityPdp = null;
 773                 if(listOfDesignated.size() > 1){
 774                         if(logger.isDebugEnabled()){
 775                                 logger.debug
 776                                 ("DesignatedWaiter.run: myPdp: {} listOfDesignated.size(): {}", myPdp.getPdpId(), listOfDesignated.size());
 777                         }
 778                         DroolsPdp rejectedPdp = null;
 779                         DroolsPdp lowestPrioritySameSite = null;
 780                         DroolsPdp lowestPriorityDifferentSite = null;
 781                         for(DroolsPdp pdp : listOfDesignated){
 782                                 // We need to determine if another PDP is the lowest priority
 783                                 if(nullSafeEquals(pdp.getSiteName(),mostRecentPrimary.getSiteName())){
 784                                         if(lowestPrioritySameSite == null){
 785                                                 if(lowestPriorityDifferentSite != null){
 786                                                         rejectedPdp = lowestPriorityDifferentSite;
 787                                                 }
 788                                                 lowestPrioritySameSite = pdp;
 789                                         }else{
 790                                                 if(pdp.getPdpId().equals((lowestPrioritySameSite.getPdpId()))){
 791                                                         continue;//nothing to compare
 792                                                 }
 793                                                 if(pdp.comparePriority(lowestPrioritySameSite) <0){
 794                                                         if(logger.isDebugEnabled()){
 795                                                                 logger.debug
 796                                                                 ("\nDesignatedWaiter.run: myPdp {}  listOfDesignated pdp ID: {}"
 797                                                                         + " has lower priority than pdp ID: {}",myPdp.getPdpId(), pdp.getPdpId(),
 798                                                                         lowestPrioritySameSite.getPdpId());
 799                                                         }
 800                                                         //we need to reject lowestPrioritySameSite
 801                                                         rejectedPdp = lowestPrioritySameSite;
 802                                                         lowestPrioritySameSite = pdp;
 803                                                 } else{
 804                                                         //we need to reject pdp and keep lowestPrioritySameSite
 805                                                         if(logger.isDebugEnabled()){
 806                                                                 logger.debug
 807                                                                 ("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {} "
 808                                                                         + " has higher priority than pdp ID: {}", myPdp.getPdpId(),pdp.getPdpId(),
 809                                                                         lowestPrioritySameSite.getPdpId());
 810                                                         }
 811                                                         rejectedPdp = pdp;
 812                                                 }
 813                                         }
 814                                 } else{
 815                                         if(lowestPrioritySameSite != null){
 816                                                 //if we already have a candidate for same site, we don't want to bother with different sites
 817                                                 rejectedPdp = pdp;
 818                                         } else{
 819                                                 if(lowestPriorityDifferentSite == null){
 820                                                         lowestPriorityDifferentSite = pdp;
 821                                                         continue;
 822                                                 }
 823                                                 if(pdp.getPdpId().equals((lowestPriorityDifferentSite.getPdpId()))){
 824                                                         continue;//nothing to compare
 825                                                 }
 826                                                 if(pdp.comparePriority(lowestPriorityDifferentSite) <0){
 827                                                         if(logger.isDebugEnabled()){
 828                                                                 logger.debug
 829                                                                 ("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 830                                                                         + " has lower priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 831                                                                         lowestPriorityDifferentSite.getPdpId());
 832                                                         }
 833                                                         //we need to reject lowestPriorityDifferentSite
 834                                                         rejectedPdp = lowestPriorityDifferentSite;
 835                                                         lowestPriorityDifferentSite = pdp;
 836                                                 } else{
 837                                                         //we need to reject pdp and keep lowestPriorityDifferentSite
 838                                                         if(logger.isDebugEnabled()){
 839                                                                 logger.debug
 840                                                                 ("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 841                                                                         + " has higher priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 842                                                                         lowestPriorityDifferentSite.getPdpId());
 843                                                         }
 844                                                         rejectedPdp = pdp;
 845                                                 }
 846                                         }
 847                                 }
 848                                 // If the rejectedPdp is myPdp, we need to stand it down and demote it.  Each pdp is responsible
 849                                 // for demoting itself
 850                                 if(rejectedPdp != null && nullSafeEquals(rejectedPdp.getPdpId(),myPdp.getPdpId())){
 851                                         if(logger.isDebugEnabled()){
 852                                                 logger.debug
 853                                                 ("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated myPdp ID: {}"
 854                                                         + " is NOT the lowest priority.  Executing stateManagement.demote()\n\n", myPdp.getPdpId(),
 855                                                          myPdp.getPdpId());
 856                                         }
 857                                         // We found that myPdp is on the listOfDesignated and it is not the lowest priority
 858                                         // So, we must demote it
 859                                         try {
 860                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 861                                                 myPdp.setDesignated(false);
 862                                                 pdpsConnector.setDesignated(myPdp, false);
 863                                                 isDesignated = false;
 864                                                 String standbyStatus = stateManagementFeature.getStandbyStatus();
 865                                                 if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 866                                                                 standbyStatus.equals(StateManagement.COLD_STANDBY))){
 867                                                         /*
 868                                                          * Only call demote if it is not already in the right state.  Don't worry about
 869                                                          * synching the lower level topic endpoint states.  That is done by the
 870                                                          * refreshStateAudit.
 871                                                          */
 872                                                         stateManagementFeature.demote();
 873                                                 }
 874                                         } catch (Exception e) {
 875                                                 myPdp.setDesignated(false);
 876                                                 pdpsConnector.setDesignated(myPdp, false);
 877                                                 isDesignated = false;
 878                                                 logger.error
 879                                                 ("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 880                                                                 + "demote myPdp {} myPdp.getPdpId(), message= {}", myPdp.getPdpId(),
 881                                                                 e);
 882                                         }
 883                                 }
 884                         } //end: for(DroolsPdp pdp : listOfDesignated)
 885                         if(lowestPrioritySameSite != null){
 886                                 lowestPriorityPdp = lowestPrioritySameSite;
 887                         } else {
 888                                 lowestPriorityPdp = lowestPriorityDifferentSite;
 889                         }
 890                         //now we have a valid value for lowestPriorityPdp
 891                         if(logger.isDebugEnabled()){
 892                                 logger.debug
 893                                 ("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 894                                         + "found the LOWEST priority pdp ID: {} "
 895                                         + " It is now the designatedPpd from the perspective of myPdp ID: {} \n\n",
 896                                          myPdp.getPdpId(), lowestPriorityPdp.getPdpId(), myPdp);
 897                         }
 898                         designatedPdp = lowestPriorityPdp;
 899
 900                 } else if(listOfDesignated.isEmpty()){
 901                         if(logger.isDebugEnabled()){
 902                                 logger.debug
 903                                 ("\nDesignatedWaiter.run: myPdp: {} listOfDesignated is: EMPTY.", myPdp.getPdpId());
 904                         }
 905                         designatedPdp = null;
 906                 } else{ //only one in listOfDesignated
 907                         if(logger.isDebugEnabled()){
 908                                 logger.debug
 909                                 ("\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 910                                         + "has ONE entry. PDP ID: {}", myPdp.getPdpId(), listOfDesignated.get(0).getPdpId());
 911                         }
 912                         designatedPdp = listOfDesignated.get(0);
 913                 }
 914                 return designatedPdp;
 915
 916         }
 917
 918         private class TimerUpdateClass extends TimerTask{
 919
 920                 @Override
 921                 public void run() {
 922                         try{
 923                                 if(logger.isDebugEnabled()){
 924                                         logger.debug("TimerUpdateClass.run: entry");
 925                                 }
 926                                 checkWaitTimer();
 927                         }catch(Exception e){
 928                                 logger.error("TimerUpdateClass.run caught an unexpected exception: ", e);
 929                         }
 930                         if(logger.isDebugEnabled()){
 931                                 logger.debug("TimerUpdateClass.run.exit");
 932                         }
 933                 }
 934         }
 935         @Override
 936         public void checkThreadStatus() {
 937                 checkWaitTimer();
 938         }
 939
 940         private void checkWaitTimer(){
 941                 synchronized(checkWaitTimerLock){
 942                         try{
 943                                 if(logger.isDebugEnabled()){
 944                                         logger.debug("checkWaitTimer: entry");
 945                                 }
 946                                 Date now = new Date();
 947                                 long nowMs = now.getTime();
 948                                 long waitTimerMs = waitTimerLastRunDate.getTime();
 949
 950                                 //give it 10 times leeway
 951                                 if((nowMs - waitTimerMs)  > 10*pdpUpdateInterval){
 952                                         if(allSeemsWell==null || allSeemsWell){
 953                                                 allSeemsWell = false;
 954                                                 if(logger.isDebugEnabled()){
 955                                                         logger.debug("checkWaitTimer: calling allSeemsWell with ALLNOTWELL param");
 956                                                 }
 957                                                 stateManagementFeature.allSeemsWell(this.getClass().getName(),
 958                                                                 StateManagementFeatureAPI.ALLNOTWELL,
 959                                                                 "DesignationWaiter/ElectionHandler has STALLED");
 960                                         }
 961                                         logger.error("checkWaitTimer: nowMs - waitTimerMs = {}"
 962                                                         + ", exceeds 10* pdpUpdateInterval = {}"
 963                                                         + " DesignationWaiter is STALLED!", (nowMs - waitTimerMs), (10*pdpUpdateInterval));
 964                                 }else if(allSeemsWell==null || !allSeemsWell){
 965                                         allSeemsWell = true;
 966                                         stateManagementFeature.allSeemsWell(this.getClass().getName(),
 967                                                         StateManagementFeatureAPI.ALLSEEMSWELL,
 968                                                         "DesignationWaiter/ElectionHandler has RESUMED");
 969                                         logger.info("DesignationWaiter/ElectionHandler has RESUMED");
 970                                 }
 971                                 if(logger.isDebugEnabled()){
 972                                         logger.debug("checkWaitTimer: exit");
 973                                 }
 974                         }catch(Exception e){
 975                                 logger.error("checkWaitTimer: caught unexpected exception: ", e);
 976                         }
 977                 }
 978         }
 979
 980         private long getDWaiterStartMs(){
 981                 Date now = new Date();
 982
 983                 // Retrieve the ms since the epoch
 984                 long nowMs = now.getTime();
 985
 986                 // Time since the end of the last pdpUpdateInterval multiple
 987                 long nowModMs = nowMs % pdpUpdateInterval;
 988
 989                 // Time to the start of the next pdpUpdateInterval multiple
 990                 long startMs = 2*pdpUpdateInterval - nowModMs;
 991
 992                 // Give the start time a minimum of a 5 second cushion
 993                 if(startMs < 5000){
 994                         // Start at the beginning  of following interval
 995                         startMs = pdpUpdateInterval + startMs;
 996                 }
 997                 return startMs;
 998         }
 999
1000         private boolean nullSafeEquals(Object one, Object two){
1001                 if(one == null && two == null){
1002                         return true;
1003                 }
1004                 if(one != null && two != null){
1005                         return one.equals(two);
1006                 }
1007                 return false;
1008         }
1009
1010         public String getPdpdNowActive(){
1011                 return pdpdNowActive;
1012         }
1013
1014         public String getPdpdLastActive(){
1015                 return pdpdLastActive;
1016         }
1017 }