gerrit.onap Code Review - policy/drools-pdp.git/blob

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * feature-active-standby-management
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.policy.drools.activestandby;
  22
  23 import java.util.ArrayList;
  24 import java.util.Collection;
  25 import java.util.Date;
  26 import java.util.Timer;
  27 import java.util.TimerTask;
  28
  29 import org.onap.policy.common.im.StateManagement;
  30 import org.slf4j.Logger;
  31 import org.slf4j.LoggerFactory;
  32 import org.onap.policy.drools.statemanagement.StateManagementFeatureAPI;
  33
  34 public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
  35         // get an instance of logger
  36         private final static Logger  logger = LoggerFactory.getLogger(DroolsPdpsElectionHandler.class);
  37         private DroolsPdpsConnector pdpsConnector;
  38         private Object checkWaitTimerLock = new Object();
  39         private Object designationWaiterLock = new Object();
  40
  41         /*
  42          * Must be static, so it can be referenced by JpaDroolsPdpsConnector,
  43          * without requiring a reference to the election handler instantiation.
  44          */
  45         private static DroolsPdp myPdp;
  46
  47         private DesignationWaiter designationWaiter;
  48         private Timer updateWorker;
  49         private Timer waitTimer;
  50         private Date waitTimerLastRunDate;
  51
  52         // The interval between checks of the DesignationWaiter to be sure it is running.
  53         private int pdpCheckInterval;
  54
  55         // The interval between runs of the DesignationWaiter
  56         private int pdpUpdateInterval;
  57
  58         private volatile boolean isDesignated;
  59
  60         private String pdpdNowActive;
  61         private String pdpdLastActive;
  62
  63         private Boolean allSeemsWell=true;
  64
  65         private StateManagementFeatureAPI stateManagementFeature;
  66
  67         private static boolean isUnitTesting = false;
  68         public static void setIsUnitTesting(boolean val){
  69                 isUnitTesting = val;
  70         }
  71         private static boolean isStalled = false;
  72         public static void setIsStalled(boolean val){
  73                 isStalled = val;
  74         }
  75
  76         public DroolsPdpsElectionHandler(DroolsPdpsConnector pdps, DroolsPdp myPdp){
  77                 if (pdps == null) {
  78                         logger.error("DroolsPdpsElectinHandler(): pdpsConnector==null");
  79                         throw new IllegalArgumentException("DroolsPdpsElectinHandler(): pdpsConnector==null");
  80                 }
  81                 if (myPdp == null){
  82                         logger.error("DroolsPdpsElectinHandler(): droolsPdp==null");
  83                         throw new IllegalArgumentException("DroolsPdpsElectinHandler(): DroolsPdp==null");
  84                 }
  85
  86                 pdpdNowActive = null;
  87                 pdpdLastActive = null;
  88                 this.pdpsConnector = pdps;
  89                 DroolsPdpsElectionHandler.myPdp = myPdp;
  90                 this.isDesignated = false;
  91                 pdpCheckInterval = 3000;
  92                 try{
  93                         pdpCheckInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(ActiveStandbyProperties.PDP_CHECK_INVERVAL));
  94                 }catch(Exception e){
  95                         logger.error
  96                         ("Could not get pdpCheckInterval property. Using default {}",pdpCheckInterval, e);
  97                 }
  98                 pdpUpdateInterval = 2000;
  99                 try{
 100                         pdpUpdateInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(ActiveStandbyProperties.PDP_UPDATE_INTERVAL));
 101                 }catch(Exception e){
 102                         logger.error
 103                         ("Could not get pdpUpdateInterval property. Using default {} ", pdpUpdateInterval, e);
 104                 }
 105
 106                 Date now = new Date();
 107
 108                 // Retrieve the ms since the epoch
 109                 long nowMs = now.getTime();
 110
 111                 // Create the timer which will update the updateDate in DroolsPdpEntity table.
 112                 // This is the heartbeat
 113                 updateWorker = new Timer();
 114
 115                 // Schedule the TimerUpdateClass to run at 100 ms and run at pdpCheckInterval ms thereafter
 116                 // NOTE: The first run of the TimerUpdateClass results in myPdp being added to the
 117                 // drools droolsPdpEntity table.
 118                 updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
 119
 120                 // Create the timer which will run the election algorithm
 121                 waitTimer = new Timer();
 122
 123                 // Schedule it to start in startMs ms (so it will run after the updateWorker and run at pdpUpdateInterval ms thereafter
 124                 long startMs = getDWaiterStartMs();
 125                 designationWaiter = new DesignationWaiter();
 126                 waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
 127                 waitTimerLastRunDate = new Date(nowMs + startMs);
 128
 129                 //Get the StateManagementFeature instance
 130
 131                 for (StateManagementFeatureAPI feature : StateManagementFeatureAPI.impl.getList())
 132                 {
 133                         if (feature.getResourceName().equals(myPdp.getPdpId()))
 134                         {
 135                                 if(logger.isDebugEnabled()){
 136                                         logger.debug("DroolsPdpsElectionHandler: Found StateManagementFeature"
 137                                                 + " with resourceName: {}", myPdp.getPdpId());
 138                                 }
 139                                 stateManagementFeature = feature;
 140                                 break;
 141                         }
 142                 }
 143                 if(stateManagementFeature == null){
 144                         logger.error("DroolsPdpsElectionHandler failed to initialize.  "
 145                                         + "Unable to get instance of StateManagementFeatureAPI "
 146                                         + "with resourceID: {}", myPdp.getPdpId());
 147                 }
 148         }
 149
 150         /*
 151          * When the JpaDroolsPdpsConnector.standDown() method is invoked, it needs
 152          * access to myPdp, so it can keep its designation status in sync with the
 153          * DB.
 154          */
 155         public static void setMyPdpDesignated(boolean designated) {
 156                 if(logger.isDebugEnabled()){
 157                         logger.debug
 158                         ("setMyPdpDesignated: designated= {}", designated);
 159                 }
 160                 myPdp.setDesignated(designated);
 161         }
 162
 163         private class DesignationWaiter extends TimerTask {
 164                 // get an instance of logger
 165                 private final Logger  logger = LoggerFactory.getLogger(DesignationWaiter.class);
 166
 167                 @Override
 168                 public void run() {
 169                         try{
 170                                 if(logger.isDebugEnabled()){
 171                                         logger.debug
 172                                         ("DesignatedWaiter.run: Entering");
 173                                 }
 174
 175                                 //This is for testing the checkWaitTimer
 176                                 if(isUnitTesting && isStalled){
 177                                         if(logger.isDebugEnabled()){
 178                                                 logger.debug("DesignatedWaiter.run: isUnitTesting = {} isStalled = {}", isUnitTesting, isStalled);
 179                                         }
 180                                         return;
 181                                 }
 182
 183                                 synchronized (designationWaiterLock) {
 184
 185                                         if(logger.isDebugEnabled()){
 186                                                 logger.debug
 187                                                 ("DesignatedWaiter.run: Entering synchronized block");
 188                                         }
 189
 190                                         //It is possible that multiple PDPs are designated lead.  So, we will make a list of all designated
 191                                         //PDPs and then decide which one really should be designated at the end.
 192                                         ArrayList<DroolsPdp> listOfDesignated = new ArrayList<DroolsPdp>();
 193
 194                                         Collection<DroolsPdp> pdps = pdpsConnector.getDroolsPdps();
 195                                         DroolsPdp designatedPdp = null;
 196
 197                                         if(logger.isDebugEnabled()){
 198                                                 logger.debug
 199                                                 ("DesignatedWaiter.run: pdps.size= {}", pdps.size());
 200                                         }
 201
 202                                         //This is only true if all designated PDPs have failed
 203                                         boolean designatedPdpHasFailed = pdpsConnector.hasDesignatedPdpFailed(pdps);
 204                                         if(logger.isDebugEnabled()){
 205                                                 logger.debug
 206                                                 ("DesignatedWaiter.run: designatedPdpHasFailed= {}", designatedPdpHasFailed);
 207                                         }
 208                                         for (DroolsPdp pdp : pdps) {
 209                                                 if(logger.isDebugEnabled()){
 210                                                         logger.debug
 211                                                         ("DesignatedWaiter.run: evaluating pdp ID: {}", pdp.getPdpId());
 212                                                 }
 213
 214                                                 /*
 215                                                  * Note: side effect of isPdpCurrent is that any stale but
 216                                                  * designated PDPs will be marked as un-designated.
 217                                                  */
 218                                                 boolean isCurrent = pdpsConnector.isPdpCurrent(pdp);
 219
 220                                                 /*
 221                                                  * We can't use stateManagement.getStandbyStatus() here, because
 222                                                  * we need the standbyStatus, not for this PDP, but for the PDP
 223                                                  * being processed by this loop iteration.
 224                                                  */
 225                                                 String standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 226                                                 if(standbyStatus==null){
 227                                                         // Treat this case as a cold standby -- if we
 228                                                         // abort here, no sessions will be created in a
 229                                                         // single-node test environment.
 230                                                         standbyStatus = StateManagement.COLD_STANDBY;
 231                                                 }
 232                                                 if(logger.isDebugEnabled()){
 233                                                         logger.debug
 234                                                         ("DesignatedWaiter.run: PDP= {},  isCurrent= {}", pdp.getPdpId(), isCurrent);
 235                                                 }
 236
 237                                                 /*
 238                                                  * There are 4 combinations of isDesignated and isCurrent.  We will examine each one in-turn
 239                                                  * and evaluate the each pdp in the list of pdps against each combination.
 240                                                  *
 241                                                  * This is the first combination of isDesignated and isCurrent
 242                                                  */
 243                                                 if (pdp.isDesignated()  &&  isCurrent) {
 244                                                         //It is current, but it could have a standbystatus=coldstandby / hotstandby
 245                                                         //If so, we need to stand it down and demote it
 246                                                         if(!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)){
 247                                                                 if(pdp.getPdpId().equals(myPdp.getPdpId())){
 248                                                                         if(logger.isDebugEnabled()){
 249                                                                                 logger.debug
 250                                                                                 ("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 251                                                                                         + "butstandbystatus is not providingservice. "
 252                                                                                         + " Executing stateManagement.demote()" + "\n\n", myPdp.getPdpId());
 253                                                                         }
 254                                                                         // So, we must demote it
 255                                                                         try {
 256                                                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 257                                                                                 //This will change isDesignated and it can enter another if(combination) below
 258                                                                                 pdpsConnector.standDownPdp(pdp.getPdpId());
 259                                                                                 myPdp.setDesignated(false);
 260                                                                                 isDesignated = false;
 261                                                                                 if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 262                                                                                                 standbyStatus.equals(StateManagement.COLD_STANDBY))){
 263                                                                                         /*
 264                                                                                          * Only demote it if it appears it has not already been demoted. Don't worry
 265                                                                                          * about synching with the topic endpoint states.  That is done by the
 266                                                                                          * refreshStateAudit
 267                                                                                          */
 268                                                                                         stateManagementFeature.demote();
 269                                                                                 }
 270                                                                                 //update the standbystatus to check in a later combination of isDesignated and isCurrent
 271                                                                                 standbyStatus=stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 272                                                                         } catch (Exception e) {
 273                                                                                 logger.error
 274                                                                                 ("DesignatedWaiter.run: myPdp: {} "
 275                                                                                                 + "Caught Exception attempting to demote myPdp,"
 276                                                                                                 + "message= {}", myPdp.getPdpId(), e);
 277                                                                         }
 278                                                                 }else{
 279                                                                         // Don't demote a remote PDP that is current.  It should catch itself
 280                                                                         if(logger.isDebugEnabled()){
 281                                                                                 logger.debug
 282                                                                                 ("\n\nDesignatedWaiter.run: myPdp {} is current and designated, "
 283                                                                                         + "but standbystatus is not providingservice. "
 284                                                                                         + " Cannot execute stateManagement.demote() since it it is not myPdp\n\n", myPdp.getPdpId());
 285                                                                         }
 286                                                                 }
 287
 288                                                         }else{
 289                                                                 // If we get here, it is ok to be on the list
 290                                                                 if(logger.isDebugEnabled()){
 291                                                                         logger.debug
 292                                                                         ("DesignatedWaiter.run: PDP= {} is designated, current and {} Noting PDP as "
 293                                                                                 + "designated, standbyStatus= {}", pdp.getPdpId(), standbyStatus, standbyStatus);
 294                                                                 }
 295                                                                 listOfDesignated.add(pdp);
 296                                                         }
 297
 298
 299                                                 }
 300
 301
 302                                                 /*
 303                                                  * The second combination of isDesignated and isCurrent
 304                                                  *
 305                                                  * PDP is designated but not current; it has failed.   So we stand it down (it doesn't matter what
 306                                                  * its standbyStatus is). None of these go on the list.
 307                                                  */
 308                                                 if (pdp.isDesignated()  &&  !isCurrent) {
 309                                                         if(logger.isDebugEnabled()){
 310                                                                 logger.debug
 311                                                                 ("INFO: DesignatedWaiter.run: PDP= {} is currently designated but is not current; "
 312                                                                         + "it has failed.  Standing down.  standbyStatus= {}", pdp.getPdpId(), standbyStatus);
 313                                                         }
 314                                                         /*
 315                                                          * Changes designated to 0 but it is still potentially providing service
 316                                                          * Will affect isDesignated, so, it can enter an if(combination) below
 317                                                          */
 318                                                         pdpsConnector.standDownPdp(pdp.getPdpId());
 319
 320                                                         //need to change standbystatus to coldstandby
 321                                                         if (pdp.getPdpId().equals(myPdp.getPdpId())){
 322                                                                 if(logger.isDebugEnabled()){
 323                                                                         logger.debug
 324                                                                         ("\n\nDesignatedWaiter.run: myPdp {} is not Current. "
 325                                                                                 + " Executing stateManagement.disableFailed()\n\n", myPdp.getPdpId());
 326                                                                 }
 327                                                                 // We found that myPdp is designated but not current
 328                                                                 // So, we must cause it to disableFail
 329                                                                 try {
 330                                                                         myPdp.setDesignated(false);
 331                                                                         pdpsConnector.setDesignated(myPdp, false);
 332                                                                         isDesignated = false;
 333                                                                         stateManagementFeature.disableFailed();
 334                                                                 } catch (Exception e) {
 335                                                                         logger.error
 336                                                                         ("DesignatedWaiter.run: myPdp: {} Caught Exception "
 337                                                                                         + "attempting to disableFail myPdp {}, message= {}",
 338                                                                                         myPdp.getPdpId(), myPdp.getPdpId(), e);
 339                                                                 }
 340                                                         } else { //it is a remote PDP that is failed
 341                                                                 if(logger.isDebugEnabled()){
 342                                                                         logger.debug
 343                                                                         ("\n\nDesignatedWaiter.run: PDP {} is not Current. "
 344                                                                                 + " Executing stateManagement.disableFailed(otherResourceName)\n\n", pdp.getPdpId() );
 345                                                                 }
 346                                                                 // We found a PDP is designated but not current
 347                                                                 // We already called standdown(pdp) which will change designated to false
 348                                                                 // Now we need to disableFail it to get its states in synch.  The standbyStatus
 349                                                                 // should equal coldstandby
 350                                                                 try {
 351                                                                         stateManagementFeature.disableFailed(pdp.getPdpId());
 352                                                                 } catch (Exception e) {
 353                                                                         logger.error
 354                                                                         ("DesignatedWaiter.run: for PDP {}  Caught Exception attempting to "
 355                                                                                         + "disableFail({}), message= {}",
 356                                                                                         pdp.getPdpId(), pdp.getPdpId(), e);
 357                                                                 }
 358
 359                                                         }
 360                                                         continue; //we are not going to do anything else with this pdp
 361                                                 }
 362
 363                                                 /*
 364                                                  * The third combination of isDesignated and isCurrent
 365                                                  * /*
 366                                                  * If a PDP is not currently designated but is providing service (erroneous, but recoverable) or hot standby
 367                                                  * we can add it to the list of possible designated if all the designated have failed
 368                                                  */
 369                                                 if (!pdp.isDesignated() && isCurrent){
 370                                                         if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 371                                                                         standbyStatus.equals(StateManagement.COLD_STANDBY))){
 372                                                                 if(logger.isDebugEnabled()){
 373                                                                         logger.debug("\n\nDesignatedWaiter.run: PDP {}"
 374                                                                                 + " is NOT designated but IS current and"
 375                                                                                 + " has a standbystatus= {}", pdp.getPdpId(), standbyStatus);
 376                                                                 }
 377                                                                 // Since it is current, we assume it can adjust its own state.
 378                                                                 // We will demote if it is myPdp
 379                                                                 if(pdp.getPdpId().equals(myPdp.getPdpId())){
 380                                                                         //demote it
 381                                                                         if(logger.isDebugEnabled()){
 382                                                                                 logger.debug("DesignatedWaiter.run: PDP {} going to "
 383                                                                                         + "setDesignated = false and calling stateManagement.demote",  pdp.getPdpId());
 384                                                                         }
 385                                                                         try {
 386                                                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 387                                                                                 pdpsConnector.setDesignated(myPdp, false);
 388                                                                                 myPdp.setDesignated(false);
 389                                                                                 isDesignated = false;
 390                                                                                 //This is definitely not a redundant call.  It is attempting to correct a problem
 391                                                                                 stateManagementFeature.demote();
 392                                                                                 //recheck the standbystatus
 393                                                                                 standbyStatus = stateManagementFeature.getStandbyStatus(pdp.getPdpId());
 394                                                                         } catch (Exception e) {
 395                                                                                 logger.error
 396                                                                                 ("DesignatedWaiter.run: myPdp: {} Caught Exception "
 397                                                                                                 + "attempting to demote myPdp {}, message = {}",  myPdp.getPdpId(),
 398                                                                                                 myPdp.getPdpId(), e);
 399                                                                         }
 400
 401                                                                 }
 402                                                         }
 403                                                         if(standbyStatus.equals(StateManagement.HOT_STANDBY) && designatedPdpHasFailed){
 404                                                                 //add it to the list
 405                                                                 if(logger.isDebugEnabled()){
 406                                                                         logger.debug
 407                                                                         ("INFO: DesignatedWaiter.run: PDP= {}"
 408                                                                                 + " is not designated but is {} and designated PDP "
 409                                                                                 + "has failed.  standbyStatus= {}", pdp.getPdpId(),
 410                                                                                 standbyStatus, standbyStatus);
 411                                                                 }
 412                                                                 listOfDesignated.add(pdp);
 413                                                         }
 414                                                         continue; //done with this one
 415                                                 }
 416
 417                                                 /*
 418                                                  * The fourth combination of isDesignated and isCurrent
 419                                                  *
 420                                                  * We are not going to put any of these on the list since it appears they have failed.
 421
 422                                                  *
 423                                                  */
 424                                                 if(!pdp.isDesignated() && !isCurrent) {
 425                                                         if(logger.isDebugEnabled()){
 426                                                                 logger.debug
 427                                                                 ("INFO: DesignatedWaiter.run: PDP= {} "
 428                                                                         + "designated= {}, current= {}, "
 429                                                                         + "designatedPdpHasFailed= {}, "
 430                                                                         + "standbyStatus= {}",pdp.getPdpId(),
 431                                                                         pdp.isDesignated(), isCurrent, designatedPdpHasFailed, standbyStatus);
 432                                                         }
 433                                                         if(!standbyStatus.equals(StateManagement.COLD_STANDBY)){
 434                                                                 //stand it down
 435                                                                 //disableFail it
 436                                                                 pdpsConnector.standDownPdp(pdp.getPdpId());
 437                                                                 if(pdp.getPdpId().equals(myPdp.getPdpId())){
 438                                                                         /*
 439                                                                          * I don't actually know how this condition could happen, but if it did, we would want
 440                                                                          * to declare it failed.
 441                                                                          */
 442                                                                         if(logger.isDebugEnabled()){
 443                                                                                 logger.debug
 444                                                                                 ("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 445                                                                                         + " Executing stateManagement.disableFailed()\n\n", myPdp.getPdpId());
 446                                                                         }
 447                                                                         // So, we must disableFail it
 448                                                                         try {
 449                                                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 450                                                                                 pdpsConnector.setDesignated(myPdp, false);
 451                                                                                 myPdp.setDesignated(false);
 452                                                                                 isDesignated = false;
 453                                                                                 stateManagementFeature.disableFailed();
 454                                                                         } catch (Exception e) {
 455                                                                                 logger.error
 456                                                                                 ("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 457                                                                                                 + "disableFail myPdp {}, message= {}",
 458                                                                                                  myPdp.getPdpId(), myPdp.getPdpId(), e);
 459                                                                         }
 460                                                                 }else{//it is remote
 461                                                                         if(logger.isDebugEnabled()){
 462                                                                                 logger.debug
 463                                                                                 ("\n\nDesignatedWaiter.run: myPdp {} is !current and !designated, "
 464                                                                                         + " Executing stateManagement.disableFailed({})\n\n",
 465                                                                                         myPdp.getPdpId(), pdp.getPdpId());
 466                                                                         }
 467                                                                         // We already called standdown(pdp) which will change designated to false
 468                                                                         // Now we need to disableFail it to get its states in sync.  StandbyStatus = coldstandby
 469                                                                         try {
 470                                                                                 stateManagementFeature.disableFailed(pdp.getPdpId());
 471                                                                         } catch (Exception e) {
 472                                                                                 logger.error
 473                                                                                 ("DesignatedWaiter.run: for PDP {}"
 474                                                                                                 + " Caught Exception attempting to disableFail({})"
 475                                                                                                 + ", message=", pdp.getPdpId(), pdp.getPdpId(), e);
 476                                                                         }
 477                                                                 }
 478                                                         }
 479                                                 }
 480
 481
 482                                         } // end pdps loop
 483
 484                                         /*
 485                                          * We have checked the four combinations of isDesignated and isCurrent.  Where appropriate,
 486                                          * we added the PDPs to the potential list of designated pdps
 487                                          *
 488                                          * We need to give priority to pdps on the same site that is currently being used
 489                                          * First, however, we must sanitize the list of designated to make sure their are
 490                                          * only designated members or non-designated members.  There should not be both in
 491                                          * the list. Because there are real time delays, it is possible that both types could
 492                                          * be on the list.
 493                                          */
 494
 495                                         listOfDesignated = santizeDesignatedList(listOfDesignated);
 496
 497                                         /*
 498                                          * We need to figure out the last pdp that was the primary so we can get the last site
 499                                          * name and the last session numbers.  We need to create a "dummy" droolspdp since
 500                                          * it will be used in later comparisons and cannot be null.
 501                                          */
 502
 503                                         DroolsPdp mostRecentPrimary = computeMostRecentPrimary(pdps, listOfDesignated);
 504
 505                                         if(mostRecentPrimary != null){
 506                                                 pdpdLastActive = mostRecentPrimary.getPdpId();
 507                                         }
 508
 509
 510                                         /*
 511                                          * It is possible to get here with more than one pdp designated and providingservice. This normally
 512                                          * occurs when there is a race condition with multiple nodes coming up at the same time. If that is
 513                                          * the case we must determine which one is the one that should be designated and which one should
 514                                          * be demoted.
 515                                          *
 516                                          * It is possible to have 0, 1, 2 or more but not all, or all designated.
 517                                          *   If we have one designated and current, we chose it and are done
 518                                          *   If we have 2 or more, but not all, we must determine which one is in the same site as
 519                                          *   the previously designated pdp.
 520                                          */
 521
 522                                         designatedPdp = computeDesignatedPdp(listOfDesignated, mostRecentPrimary);
 523                                         if(designatedPdp != null){
 524                                                 pdpdNowActive = designatedPdp.getPdpId();
 525                                         }
 526
 527                                         if (designatedPdp == null) {
 528                                                 logger.warn
 529                                                 ("WARNING: DesignatedWaiter.run: No viable PDP found to be Designated. designatedPdp still null.");
 530                                                 // Just to be sure the parameters are correctly set
 531                                                 myPdp.setDesignated(false);
 532                                                 pdpsConnector.setDesignated(myPdp,false);
 533                                                 isDesignated = false;
 534
 535                                                 waitTimerLastRunDate = new Date();
 536                                                 if(logger.isDebugEnabled()){
 537                                                         logger.debug("DesignatedWaiter.run (designatedPdp == null) waitTimerLastRunDate = {}", waitTimerLastRunDate);
 538                                                 }
 539                                                 myPdp.setUpdatedDate(waitTimerLastRunDate);
 540                                                 pdpsConnector.update(myPdp);
 541
 542                                                 return;
 543
 544                                         } else if (designatedPdp.getPdpId().equals(myPdp.getPdpId())) {
 545                                                 if(logger.isDebugEnabled()){
 546                                                         logger.debug
 547                                                         ("DesignatedWaiter.run: designatedPdp is PDP={}", myPdp.getPdpId());
 548                                                 }
 549                                                 /*
 550                                                  * update function expects myPdp.isDesignated to be true.
 551                                                  */
 552                                                 try {
 553                                                         //Keep the order like this.  StateManagement is last since it triggers controller init
 554                                                         myPdp.setDesignated(true);
 555                                                         myPdp.setDesignatedDate(new Date());
 556                                                         pdpsConnector.setDesignated(myPdp, true);
 557                                                         isDesignated = true;
 558                                                         String standbyStatus = stateManagementFeature.getStandbyStatus();
 559                                                         if(!standbyStatus.equals(StateManagement.PROVIDING_SERVICE)){
 560                                                                 /*
 561                                                                  * Only call promote if it is not already in the right state.  Don't worry about
 562                                                                  * synching the lower level topic endpoint states.  That is done by the
 563                                                                  * refreshStateAudit.
 564                                                                  * Note that we need to fetch the session list from 'mostRecentPrimary'
 565                                                                  * at this point -- soon, 'mostRecentPrimary' will be set to this host.
 566                                                                  */
 567                                                                 //this.sessions = mostRecentPrimary.getSessions();
 568                                                                 stateManagementFeature.promote();
 569                                                         }
 570                                                 } catch (Exception e) {
 571                                                         logger.error
 572                                                         ("ERROR: DesignatedWaiter.run: Caught Exception attempting to promote PDP={}"
 573                                                                         + ", message=", myPdp.getPdpId(), e);
 574                                                         myPdp.setDesignated(false);
 575                                                         pdpsConnector.setDesignated(myPdp,false);
 576                                                         isDesignated = false;
 577                                                         //If you can't promote it, demote it
 578                                                         try {
 579                                                                 String standbyStatus = stateManagementFeature.getStandbyStatus();
 580                                                                 if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 581                                                                                 standbyStatus.equals(StateManagement.COLD_STANDBY))){
 582                                                                         /*
 583                                                                          * Only call demote if it is not already in the right state.  Don't worry about
 584                                                                          * synching the lower level topic endpoint states.  That is done by the
 585                                                                          * refreshStateAudit.
 586                                                                          */
 587                                                                         stateManagementFeature.demote();
 588                                                                 }
 589                                                         } catch (Exception e1) {
 590                                                                 logger.error
 591                                                                 ("ERROR: DesignatedWaiter.run: Caught StandbyStatusException "
 592                                                                                 + "attempting to promote then demote PDP={}, message=",
 593                                                                                 myPdp.getPdpId(), e1);
 594                                                         }
 595
 596                                                 }
 597                                                 waitTimerLastRunDate = new Date();
 598                                                 if(logger.isDebugEnabled()){
 599                                                         logger.debug("DesignatedWaiter.run (designatedPdp.getPdpId().equals(myPdp.getPdpId())) "
 600                                                                 + "waitTimerLastRunDate = " + waitTimerLastRunDate);
 601                                                 }
 602                                                 myPdp.setUpdatedDate(waitTimerLastRunDate);
 603                                                 pdpsConnector.update(myPdp);
 604
 605                                                 return;
 606                                         }
 607                                         isDesignated = false;
 608
 609                                 } // end synchronized
 610                                 if(logger.isDebugEnabled()){
 611                                         logger.debug
 612                                         ("DesignatedWaiter.run: myPdp: {}; Returning, isDesignated= {}",
 613                                                 isDesignated, myPdp.getPdpId());
 614                                 }
 615
 616                                 Date tmpDate = new Date();
 617                                 if(logger.isDebugEnabled()){
 618                                         logger.debug("DesignatedWaiter.run (end of run) waitTimerLastRunDate = {}", tmpDate);
 619                                 }
 620
 621                                 waitTimerLastRunDate = tmpDate;
 622                                 myPdp.setUpdatedDate(waitTimerLastRunDate);
 623                                 pdpsConnector.update(myPdp);
 624
 625                         }catch(Exception e){
 626                                 logger.error("DesignatedWaiter.run caught an unexpected exception: ", e);
 627                         }
 628                 } // end run
 629         }
 630
 631         public ArrayList<DroolsPdp> santizeDesignatedList(ArrayList<DroolsPdp> listOfDesignated){
 632
 633                 boolean containsDesignated = false;
 634                 boolean containsHotStandby = false;
 635                 ArrayList<DroolsPdp> listForRemoval = new ArrayList<DroolsPdp>();
 636                 for(DroolsPdp pdp : listOfDesignated){
 637                         if(logger.isDebugEnabled()){
 638                                 logger.debug
 639                                 ("DesignatedWaiter.run sanitizing: pdp = {}"
 640                                         + " isDesignated = {}",pdp.getPdpId(), pdp.isDesignated());
 641                         }
 642                         if(pdp.isDesignated()){
 643                                 containsDesignated = true;
 644                         }else {
 645                                 containsHotStandby = true;
 646                                 listForRemoval.add(pdp);
 647                         }
 648                 }
 649                 if(containsDesignated && containsHotStandby){
 650                         //remove the hot standby from the list
 651                         listOfDesignated.removeAll(listForRemoval);
 652                         containsHotStandby = false;
 653                 }
 654                 return listOfDesignated;
 655         }
 656
 657         public DroolsPdp computeMostRecentPrimary(Collection<DroolsPdp> pdps, ArrayList<DroolsPdp> listOfDesignated){
 658                 boolean containsDesignated = false;
 659                 for(DroolsPdp pdp : listOfDesignated){
 660                         if(pdp.isDesignated()){
 661                                 containsDesignated = true;
 662                         }
 663                 }
 664                 DroolsPdp mostRecentPrimary = new DroolsPdpImpl(null, true, 1, new Date(0));
 665                 mostRecentPrimary.setSiteName(null);
 666                 if(logger.isDebugEnabled()){
 667                         logger.debug
 668                         ("DesignatedWaiter.run listOfDesignated.size() = {}", listOfDesignated.size());
 669                 }
 670                 if(listOfDesignated.size() <=1){
 671                         if(logger.isDebugEnabled()){
 672                                 logger.debug("DesignatedWainter.run: listOfDesignated.size <=1");
 673                         }
 674                         //Only one or none is designated or hot standby.  Choose the latest designated date
 675                         for(DroolsPdp pdp : pdps){
 676                                 if(logger.isDebugEnabled()){
 677                                         logger.debug
 678                                         ("DesignatedWaiter.run pdp = {}"
 679                                                 + " pdp.getDesignatedDate() = {}", pdp.getPdpId(), pdp.getDesignatedDate());
 680                                 }
 681                                 if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 682                                         mostRecentPrimary = pdp;
 683                                         if(logger.isDebugEnabled()){
 684                                                 logger.debug
 685                                                 ("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 686                                         }
 687                                 }
 688                         }
 689                 }else if(listOfDesignated.size() == pdps.size()){
 690                         if(logger.isDebugEnabled()){
 691                                 logger.debug("DesignatedWainter.run: listOfDesignated.size = pdps.size() which is {}", pdps.size());
 692                         }
 693                         //They are all designated or all hot standby.
 694                         mostRecentPrimary = null;
 695                         for(DroolsPdp pdp : pdps){
 696                                 if(mostRecentPrimary == null){
 697                                         mostRecentPrimary = pdp;
 698                                         continue;
 699                                 }
 700                                 if(containsDesignated){ //Choose the site of the first designated date
 701                                         if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) < 0){
 702                                                 mostRecentPrimary = pdp;
 703                                                 if(logger.isDebugEnabled()){
 704                                                         logger.debug
 705                                                         ("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 706                                                 }
 707                                         }
 708                                 }else{ //Choose the site with the latest designated date
 709                                         if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 710                                                 mostRecentPrimary = pdp;
 711                                                 if(logger.isDebugEnabled()){
 712                                                         logger.debug
 713                                                         ("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 714                                                 }
 715                                         }
 716                                 }
 717                         }
 718                 }else{
 719                         if(logger.isDebugEnabled()){
 720                                 logger.debug("DesignatedWainter.run: Some but not all are designated or hot standby. ");
 721                         }
 722                         //Some but not all are designated or hot standby.
 723                         if(containsDesignated){
 724                                 if(logger.isDebugEnabled()){
 725                                         logger.debug("DesignatedWainter.run: containsDesignated = {}", containsDesignated);
 726                                 }
 727                                 /*
 728                                  * The list only contains designated.  This is a problem.  It is most likely a race
 729                                  * condition that resulted in two thinking they should be designated. Choose the
 730                                  * site with the latest designated date for the pdp not included on the designated list.
 731                                  * This should be the site that had the last designation before this race condition
 732                                  * occurred.
 733                                  */
 734                                 for(DroolsPdp pdp : pdps){
 735                                         if(listOfDesignated.contains(pdp)){
 736                                                 continue; //Don't consider this entry
 737                                         }
 738                                         if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 739                                                 mostRecentPrimary = pdp;
 740                                                 if(logger.isDebugEnabled()){
 741                                                         logger.debug
 742                                                         ("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 743                                                 }
 744                                         }
 745                                 }
 746                         }else{
 747                                 if(logger.isDebugEnabled()){
 748                                         logger.debug("DesignatedWainter.run: containsDesignated = {}", containsDesignated);
 749                                 }
 750                                 //The list only contains hot standby. Choose the site of the latest designated date
 751                                 for(DroolsPdp pdp : pdps){
 752                                         if(pdp.getDesignatedDate().compareTo(mostRecentPrimary.getDesignatedDate()) > 0){
 753                                                 mostRecentPrimary = pdp;
 754                                                 if(logger.isDebugEnabled()){
 755                                                         logger.debug
 756                                                         ("DesignatedWaiter.run mostRecentPrimary = {}", mostRecentPrimary.getPdpId());
 757                                                 }
 758                                         }
 759                                 }
 760                         }
 761                 }
 762                 return mostRecentPrimary;
 763         }
 764
 765         public DroolsPdp computeDesignatedPdp(ArrayList<DroolsPdp> listOfDesignated, DroolsPdp mostRecentPrimary){
 766                 DroolsPdp designatedPdp = null;
 767                 DroolsPdp lowestPriorityPdp = null;
 768                 if(listOfDesignated.size() > 1){
 769                         if(logger.isDebugEnabled()){
 770                                 logger.debug
 771                                 ("DesignatedWaiter.run: myPdp: {} listOfDesignated.size(): {}", myPdp.getPdpId(), listOfDesignated.size());
 772                         }
 773                         DroolsPdp rejectedPdp = null;
 774                         DroolsPdp lowestPrioritySameSite = null;
 775                         DroolsPdp lowestPriorityDifferentSite = null;
 776                         for(DroolsPdp pdp : listOfDesignated){
 777                                 // We need to determine if another PDP is the lowest priority
 778                                 if(nullSafeEquals(pdp.getSiteName(),mostRecentPrimary.getSiteName())){
 779                                         if(lowestPrioritySameSite == null){
 780                                                 if(lowestPriorityDifferentSite != null){
 781                                                         rejectedPdp = lowestPriorityDifferentSite;
 782                                                 }
 783                                                 lowestPrioritySameSite = pdp;
 784                                         }else{
 785                                                 if(pdp.getPdpId().equals((lowestPrioritySameSite.getPdpId()))){
 786                                                         continue;//nothing to compare
 787                                                 }
 788                                                 if(pdp.comparePriority(lowestPrioritySameSite) <0){
 789                                                         if(logger.isDebugEnabled()){
 790                                                                 logger.debug
 791                                                                 ("\nDesignatedWaiter.run: myPdp {}  listOfDesignated pdp ID: {}"
 792                                                                         + " has lower priority than pdp ID: {}",myPdp.getPdpId(), pdp.getPdpId(),
 793                                                                         lowestPrioritySameSite.getPdpId());
 794                                                         }
 795                                                         //we need to reject lowestPrioritySameSite
 796                                                         rejectedPdp = lowestPrioritySameSite;
 797                                                         lowestPrioritySameSite = pdp;
 798                                                 } else{
 799                                                         //we need to reject pdp and keep lowestPrioritySameSite
 800                                                         if(logger.isDebugEnabled()){
 801                                                                 logger.debug
 802                                                                 ("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {} "
 803                                                                         + " has higher priority than pdp ID: {}", myPdp.getPdpId(),pdp.getPdpId(),
 804                                                                         lowestPrioritySameSite.getPdpId());
 805                                                         }
 806                                                         rejectedPdp = pdp;
 807                                                 }
 808                                         }
 809                                 } else{
 810                                         if(lowestPrioritySameSite != null){
 811                                                 //if we already have a candidate for same site, we don't want to bother with different sites
 812                                                 rejectedPdp = pdp;
 813                                         } else{
 814                                                 if(lowestPriorityDifferentSite == null){
 815                                                         lowestPriorityDifferentSite = pdp;
 816                                                         continue;
 817                                                 }
 818                                                 if(pdp.getPdpId().equals((lowestPriorityDifferentSite.getPdpId()))){
 819                                                         continue;//nothing to compare
 820                                                 }
 821                                                 if(pdp.comparePriority(lowestPriorityDifferentSite) <0){
 822                                                         if(logger.isDebugEnabled()){
 823                                                                 logger.debug
 824                                                                 ("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 825                                                                         + " has lower priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 826                                                                         lowestPriorityDifferentSite.getPdpId());
 827                                                         }
 828                                                         //we need to reject lowestPriorityDifferentSite
 829                                                         rejectedPdp = lowestPriorityDifferentSite;
 830                                                         lowestPriorityDifferentSite = pdp;
 831                                                 } else{
 832                                                         //we need to reject pdp and keep lowestPriorityDifferentSite
 833                                                         if(logger.isDebugEnabled()){
 834                                                                 logger.debug
 835                                                                 ("\nDesignatedWaiter.run: myPdp {} listOfDesignated pdp ID: {}"
 836                                                                         + " has higher priority than pdp ID: {}", myPdp.getPdpId(), pdp.getPdpId(),
 837                                                                         lowestPriorityDifferentSite.getPdpId());
 838                                                         }
 839                                                         rejectedPdp = pdp;
 840                                                 }
 841                                         }
 842                                 }
 843                                 // If the rejectedPdp is myPdp, we need to stand it down and demote it.  Each pdp is responsible
 844                                 // for demoting itself
 845                                 if(rejectedPdp != null && nullSafeEquals(rejectedPdp.getPdpId(),myPdp.getPdpId())){
 846                                         if(logger.isDebugEnabled()){
 847                                                 logger.debug
 848                                                 ("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated myPdp ID: {}"
 849                                                         + " is NOT the lowest priority.  Executing stateManagement.demote()\n\n", myPdp.getPdpId(),
 850                                                          myPdp.getPdpId());
 851                                         }
 852                                         // We found that myPdp is on the listOfDesignated and it is not the lowest priority
 853                                         // So, we must demote it
 854                                         try {
 855                                                 //Keep the order like this.  StateManagement is last since it triggers controller shutdown
 856                                                 myPdp.setDesignated(false);
 857                                                 pdpsConnector.setDesignated(myPdp, false);
 858                                                 isDesignated = false;
 859                                                 String standbyStatus = stateManagementFeature.getStandbyStatus();
 860                                                 if(!(standbyStatus.equals(StateManagement.HOT_STANDBY) ||
 861                                                                 standbyStatus.equals(StateManagement.COLD_STANDBY))){
 862                                                         /*
 863                                                          * Only call demote if it is not already in the right state.  Don't worry about
 864                                                          * synching the lower level topic endpoint states.  That is done by the
 865                                                          * refreshStateAudit.
 866                                                          */
 867                                                         stateManagementFeature.demote();
 868                                                 }
 869                                         } catch (Exception e) {
 870                                                 myPdp.setDesignated(false);
 871                                                 pdpsConnector.setDesignated(myPdp, false);
 872                                                 isDesignated = false;
 873                                                 logger.error
 874                                                 ("DesignatedWaiter.run: myPdp: {} Caught Exception attempting to "
 875                                                                 + "demote myPdp {} myPdp.getPdpId(), message= {}", myPdp.getPdpId(),
 876                                                                 e);
 877                                         }
 878                                 }
 879                         } //end: for(DroolsPdp pdp : listOfDesignated)
 880                         if(lowestPrioritySameSite != null){
 881                                 lowestPriorityPdp = lowestPrioritySameSite;
 882                         } else {
 883                                 lowestPriorityPdp = lowestPriorityDifferentSite;
 884                         }
 885                         //now we have a valid value for lowestPriorityPdp
 886                         if(logger.isDebugEnabled()){
 887                                 logger.debug
 888                                 ("\n\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 889                                         + "found the LOWEST priority pdp ID: {} "
 890                                         + " It is now the designatedPpd from the perspective of myPdp ID: {} \n\n",
 891                                          myPdp.getPdpId(), lowestPriorityPdp.getPdpId(), myPdp);
 892                         }
 893                         designatedPdp = lowestPriorityPdp;
 894
 895                 } else if(listOfDesignated.isEmpty()){
 896                         if(logger.isDebugEnabled()){
 897                                 logger.debug
 898                                 ("\nDesignatedWaiter.run: myPdp: {} listOfDesignated is: EMPTY.", myPdp.getPdpId());
 899                         }
 900                         designatedPdp = null;
 901                 } else{ //only one in listOfDesignated
 902                         if(logger.isDebugEnabled()){
 903                                 logger.debug
 904                                 ("\nDesignatedWaiter.run: myPdp: {} listOfDesignated "
 905                                         + "has ONE entry. PDP ID: {}", myPdp.getPdpId(), listOfDesignated.get(0).getPdpId());
 906                         }
 907                         designatedPdp = listOfDesignated.get(0);
 908                 }
 909                 return designatedPdp;
 910
 911         }
 912
 913         private class TimerUpdateClass extends TimerTask{
 914
 915                 @Override
 916                 public void run() {
 917                         try{
 918                                 if(logger.isDebugEnabled()){
 919                                         logger.debug("TimerUpdateClass.run: entry");
 920                                 }
 921                                 checkWaitTimer();
 922                         }catch(Exception e){
 923                                 logger.error("TimerUpdateClass.run caught an unexpected exception: ", e);
 924                         }
 925                         if(logger.isDebugEnabled()){
 926                                 logger.debug("TimerUpdateClass.run.exit");
 927                         }
 928                 }
 929         }
 930         @Override
 931         public void checkThreadStatus() {
 932                 checkWaitTimer();
 933         }
 934
 935         private void checkWaitTimer(){
 936                 synchronized(checkWaitTimerLock){
 937                         try{
 938                                 if(logger.isDebugEnabled()){
 939                                         logger.debug("checkWaitTimer: entry");
 940                                 }
 941                                 Date now = new Date();
 942                                 long nowMs = now.getTime();
 943                                 long waitTimerMs = waitTimerLastRunDate.getTime();
 944
 945                                 //give it 10 times leeway
 946                                 if((nowMs - waitTimerMs)  > 10*pdpUpdateInterval){
 947                                         if(allSeemsWell){
 948                                                 allSeemsWell = false;
 949                                                 if(logger.isDebugEnabled()){
 950                                                         logger.debug("checkWaitTimer: calling allSeemsWell with ALLNOTWELL param");
 951                                                 }
 952                                                 stateManagementFeature.allSeemsWell(this.getClass().getName(),
 953                                                                 StateManagementFeatureAPI.ALLNOTWELL,
 954                                                                 "DesignationWaiter/ElectionHandler has STALLED");
 955                                         }
 956                                         logger.error("checkWaitTimer: nowMs - waitTimerMs = {}"
 957                                                         + ", exceeds 10* pdpUpdateInterval = {}"
 958                                                         + " DesignationWaiter is STALLED!", (nowMs - waitTimerMs), (10*pdpUpdateInterval));
 959                                 }else if(!allSeemsWell){
 960                                         allSeemsWell = true;
 961                                         stateManagementFeature.allSeemsWell(this.getClass().getName(),
 962                                                         StateManagementFeatureAPI.ALLSEEMSWELL,
 963                                                         "DesignationWaiter/ElectionHandler has RESUMED");
 964                                         logger.info("DesignationWaiter/ElectionHandler has RESUMED");
 965                                 }
 966                                 if(logger.isDebugEnabled()){
 967                                         logger.debug("checkWaitTimer: exit");
 968                                 }
 969                         }catch(Exception e){
 970                                 logger.error("checkWaitTimer: caught unexpected exception: ", e);
 971                         }
 972                 }
 973         }
 974
 975         private long getDWaiterStartMs(){
 976                 Date now = new Date();
 977
 978                 // Retrieve the ms since the epoch
 979                 long nowMs = now.getTime();
 980
 981                 // Time since the end of the last pdpUpdateInterval multiple
 982                 long nowModMs = nowMs % pdpUpdateInterval;
 983
 984                 // Time to the start of the next pdpUpdateInterval multiple
 985                 long startMs = 2*pdpUpdateInterval - nowModMs;
 986
 987                 // Give the start time a minimum of a 5 second cushion
 988                 if(startMs < 5000){
 989                         // Start at the beginning  of following interval
 990                         startMs = pdpUpdateInterval + startMs;
 991                 }
 992                 return startMs;
 993         }
 994
 995         private boolean nullSafeEquals(Object one, Object two){
 996                 if(one == null && two == null){
 997                         return true;
 998                 }
 999                 if(one != null && two != null){
1000                         return one.equals(two);
1001                 }
1002                 return false;
1003         }
1004
1005         public String getPdpdNowActive(){
1006                 return pdpdNowActive;
1007         }
1008
1009         public String getPdpdLastActive(){
1010                 return pdpdLastActive;
1011         }
1012 }