gerrit.onap Code Review - policy/drools-pdp.git/blob

   1 /*
   2  * ============LICENSE_START=======================================================
   3  * feature-active-standby-management
   4  * ================================================================================
   5  * Copyright (C) 2017-2019 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.policy.drools.controller.test;
  22
  23 import static org.junit.Assert.assertTrue;
  24
  25 import java.io.File;
  26 import java.io.FileInputStream;
  27 import java.util.ArrayList;
  28 import java.util.Date;
  29 import java.util.List;
  30 import java.util.Properties;
  31 import javax.persistence.EntityManager;
  32 import javax.persistence.EntityManagerFactory;
  33 import javax.persistence.EntityTransaction;
  34 import javax.persistence.Persistence;
  35
  36 import org.apache.commons.lang3.time.DateUtils;
  37 import org.junit.After;
  38 import org.junit.AfterClass;
  39 import org.junit.Before;
  40 import org.junit.BeforeClass;
  41 import org.junit.Test;
  42 import org.onap.policy.common.im.AdministrativeStateException;
  43 import org.onap.policy.common.im.IntegrityMonitor;
  44 import org.onap.policy.common.im.StandbyStatusException;
  45 import org.onap.policy.common.im.StateManagement;
  46 import org.onap.policy.drools.activestandby.ActiveStandbyFeatureApi;
  47 import org.onap.policy.drools.activestandby.ActiveStandbyFeatureApiConstants;
  48 import org.onap.policy.drools.activestandby.ActiveStandbyProperties;
  49 import org.onap.policy.drools.activestandby.DroolsPdp;
  50 import org.onap.policy.drools.activestandby.DroolsPdpEntity;
  51 import org.onap.policy.drools.activestandby.DroolsPdpImpl;
  52 import org.onap.policy.drools.activestandby.DroolsPdpsConnector;
  53 import org.onap.policy.drools.activestandby.DroolsPdpsElectionHandler;
  54 import org.onap.policy.drools.activestandby.JpaDroolsPdpsConnector;
  55 import org.onap.policy.drools.activestandby.PmStandbyStateChangeNotifier;
  56 import org.onap.policy.drools.core.PolicySessionFeatureApi;
  57 import org.onap.policy.drools.statemanagement.StateManagementFeatureApi;
  58 import org.onap.policy.drools.statemanagement.StateManagementFeatureApiConstants;
  59 import org.slf4j.Logger;
  60 import org.slf4j.LoggerFactory;
  61
  62 /*
  63  * All JUnits are designed to run in the local development environment
  64  * where they have write privileges and can execute time-sensitive
  65  * tasks.
  66  *
  67  * These tests can be run as JUnits, but there is some issue with running them
  68  * as part of a "mvn install" build.  Also, they take a very long time to run
  69  * due to many real time breaks.  Consequently, they are marked as @Ignore and
  70  * only run from the desktop.
  71  */
  72
  73 public class StandbyStateManagementTest {
  74     private static final Logger  logger = LoggerFactory.getLogger(StandbyStateManagementTest.class);
  75     /*
  76      * Currently, the DroolsPdpsElectionHandler.DesignationWaiter is invoked every 1 seconds, starting
  77      * at the start of the next multiple of pdpUpdateInterval, but with a minimum of 5 sec cushion
  78      * to ensure that we wait for the DesignationWaiter to do its job, before
  79      * checking the results. Add a few seconds for safety
  80      */
  81
  82     long sleepTime = 10000;
  83
  84     /*
  85      * DroolsPdpsElectionHandler runs every 1 seconds, so a 6 second sleep should be
  86      * plenty to ensure it has time to re-promote this PDP.
  87      */
  88
  89     long electionWaitSleepTime = 6000;
  90
  91     /*
  92      * Sleep 1 seconds after each test to allow interrupt (shutdown) recovery.
  93      */
  94
  95     long interruptRecoveryTime = 5000;
  96
  97     private static EntityManagerFactory emfx;
  98     private static EntityManagerFactory emfd;
  99     private static EntityManager emx;
 100     private static EntityManager emd;
 101     private static EntityTransaction et;
 102
 103     private final String configDir = "src/test/resources";
 104
 105     /*
 106      * See the IntegrityMonitor.getJmxUrl() method for the rationale behind this jmx related processing.
 107      */
 108
 109     /**
 110      * Setup the class.
 111      *
 112      * @throws Exception exception
 113      */
 114     @BeforeClass
 115     public static void setUpClass() throws Exception {
 116
 117         String userDir = System.getProperty("user.dir");
 118         logger.debug("setUpClass: userDir={}", userDir);
 119         System.setProperty("com.sun.management.jmxremote.port", "9980");
 120         System.setProperty("com.sun.management.jmxremote.authenticate","false");
 121
 122     }
 123
 124     @AfterClass
 125     public static void tearDownClass() throws Exception {
 126     }
 127
 128     /**
 129      * Setup.
 130      *
 131      * @throws Exception exception
 132      */
 133     @Before
 134     public void setUp() throws Exception {
 135         //Create teh data access for xaml db
 136         Properties stateManagementProperties = new Properties();
 137         stateManagementProperties.load(new FileInputStream(new File(
 138                 "src/test/resources/feature-state-management.properties")));
 139
 140         emfx = Persistence.createEntityManagerFactory("junitXacmlPU", stateManagementProperties);
 141
 142         // Create an entity manager to use the DB
 143         emx = emfx.createEntityManager();
 144
 145         //Create the data access for drools db
 146         Properties activeStandbyProperties = new Properties();
 147         activeStandbyProperties.load(new FileInputStream(new File(
 148                 "src/test/resources/feature-active-standby-management.properties")));
 149
 150         emfd = Persistence.createEntityManagerFactory("junitDroolsPU", activeStandbyProperties);
 151
 152         // Create an entity manager to use the DB
 153         emd = emfd.createEntityManager();
 154     }
 155
 156     @After
 157     public void tearDown() throws Exception {
 158
 159     }
 160
 161     /**
 162      * Clean up the xacml database.
 163      *
 164      */
 165     public void cleanXacmlDb() {
 166         et = emx.getTransaction();
 167
 168         et.begin();
 169         // Make sure we leave the DB clean
 170         emx.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
 171         emx.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
 172         emx.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
 173         emx.flush();
 174         et.commit();
 175     }
 176
 177     /**
 178      * Clean up the drools db.
 179      */
 180     public void cleanDroolsDb() {
 181         et = emd.getTransaction();
 182
 183         et.begin();
 184         // Make sure we leave the DB clean
 185         emd.createQuery("DELETE FROM DroolsPdpEntity").executeUpdate();
 186         emd.flush();
 187         et.commit();
 188     }
 189
 190     /*
 191      * These JUnit tests must be run one at a time in an eclipse environment
 192      * by right-clicking StandbyStateManagementTest and selecting
 193      * "Run As" -> "JUnit Test".
 194      *
 195      * They will run successfully when you run all of them under runAllTests(),
 196      * however, you will get all sorts of non-fatal errors in the log and on the
 197      * console that result from overlapping threads that are not terminated at the
 198      * end of each test. The problem is that the JUnit environment does not terminate
 199      * all the test threads between tests. This is true even if you break each JUnit
 200      * into a separate file. Consequently, all the tests would have to be refactored
 201      * so all test object initializations are coordinated.  In other words, you
 202      * retrieve the ActiveStandbyFeature instance and other class instances only once
 203      * at the beginning of the JUnits and then reuse them throughout the tests.
 204      * Initialization of the state of the objects is pretty straight forward as it
 205      * just amounts to manipulating the entries in StateManagementEntity and
 206      * DroolsPdpEntity tables. However, some thought needs to be given to how to
 207      * "pause" the processing in ActiveStandbyFeature class.  I think we could "pause"
 208      * it by calling globalInit() which will, I think, restart it. So long as it
 209      * does not create a new instance, it will force it to go through an initialization
 210      * cycle which includes a "pause" at the beginning of proecessing.  We just must
 211      * be sure it does not create another instance - which may mean we need to add
 212      * a factory interface instead of calling the constructor directly.
 213      */
 214
 215
 216     //@Ignore
 217     @Test
 218     public void runAllTests() throws Exception {
 219         testColdStandby();
 220         testHotStandby1();
 221         testHotStandby2();
 222         testLocking1();
 223         testLocking2();
 224         testPmStandbyStateChangeNotifier();
 225         testSanitizeDesignatedList();
 226         testComputeMostRecentPrimary();
 227         testComputeDesignatedPdp();
 228     }
 229
 230     /**
 231      * Test the standby state change notifier.
 232      *
 233      * @throws Exception exception
 234      */
 235     //@Ignore
 236     //@Test
 237     public void testPmStandbyStateChangeNotifier() throws Exception {
 238         logger.debug("\n\ntestPmStandbyStateChangeNotifier: Entering\n\n");
 239         cleanXacmlDb();
 240
 241         logger.debug("testPmStandbyStateChangeNotifier: Reading activeStandbyProperties");
 242
 243         Properties activeStandbyProperties = new Properties();
 244         activeStandbyProperties.load(new FileInputStream(new File(
 245                 configDir + "/feature-active-standby-management.properties")));
 246
 247         String resourceName = "testPMS";
 248         activeStandbyProperties.setProperty("resource.name", resourceName);
 249         ActiveStandbyProperties.initProperties(activeStandbyProperties);
 250
 251         logger.debug("testPmStandbyStateChangeNotifier: Getting StateManagement instance");
 252
 253         StateManagement sm = new StateManagement(emfx, resourceName);
 254
 255         //Create an instance of the Observer
 256         PmStandbyStateChangeNotifier pmNotifier = new PmStandbyStateChangeNotifier();
 257
 258         //Register the PmStandbyStateChangeNotifier Observer
 259         sm.addObserver(pmNotifier);
 260
 261         //At this point the standbystatus = 'null'
 262         sm.lock();
 263         assertTrue(pmNotifier.getPreviousStandbyStatus().equals(StateManagement.NULL_VALUE));
 264
 265         sm.unlock();
 266         assertTrue(pmNotifier.getPreviousStandbyStatus().equals(StateManagement.NULL_VALUE));
 267
 268         //Adding standbystatus=hotstandby
 269         sm.demote();
 270         System.out.println(pmNotifier.getPreviousStandbyStatus());
 271         assertTrue(pmNotifier.getPreviousStandbyStatus().equals(
 272                 PmStandbyStateChangeNotifier.HOTSTANDBY_OR_COLDSTANDBY));
 273
 274         //Now making standbystatus=coldstandby
 275         sm.lock();
 276         assertTrue(pmNotifier.getPreviousStandbyStatus().equals(
 277                 PmStandbyStateChangeNotifier.HOTSTANDBY_OR_COLDSTANDBY));
 278
 279         //standbystatus = hotstandby
 280         sm.unlock();
 281         assertTrue(pmNotifier.getPreviousStandbyStatus().equals(
 282                 PmStandbyStateChangeNotifier.HOTSTANDBY_OR_COLDSTANDBY));
 283
 284         //standbystatus = providingservice
 285         sm.promote();
 286         //The previousStandbyStatus is not updated until after the delay activation expires
 287         assertTrue(pmNotifier.getPreviousStandbyStatus().equals(
 288                 PmStandbyStateChangeNotifier.HOTSTANDBY_OR_COLDSTANDBY));
 289
 290         //Sleep long enough for the delayActivationTimer to run
 291         sleep(5000);
 292         assertTrue(pmNotifier.getPreviousStandbyStatus().equals(StateManagement.PROVIDING_SERVICE));
 293
 294         //standbystatus = providingservice
 295         sm.promote();
 296         assertTrue(pmNotifier.getPreviousStandbyStatus().equals(StateManagement.PROVIDING_SERVICE));
 297
 298         //standbystatus = coldstandby
 299         sm.lock();
 300         assertTrue(pmNotifier.getPreviousStandbyStatus().equals(
 301                 PmStandbyStateChangeNotifier.HOTSTANDBY_OR_COLDSTANDBY));
 302
 303         //standbystatus = hotstandby
 304         sm.unlock();
 305         assertTrue(pmNotifier.getPreviousStandbyStatus().equals(
 306                 PmStandbyStateChangeNotifier.HOTSTANDBY_OR_COLDSTANDBY));
 307
 308         //standbystatus = hotstandby
 309         sm.demote();
 310         assertTrue(pmNotifier.getPreviousStandbyStatus().equals(
 311                 PmStandbyStateChangeNotifier.HOTSTANDBY_OR_COLDSTANDBY));
 312     }
 313
 314     /**
 315      * Test sanitize designated list.
 316      *
 317      * @throws Exception exception
 318      */
 319     //@Ignore
 320     //@Test
 321     public void testSanitizeDesignatedList() throws Exception {
 322
 323         logger.debug("\n\ntestSanitizeDesignatedList: Entering\n\n");
 324
 325         // Get a DroolsPdpsConnector
 326
 327         logger.debug("testSanitizeDesignatedList: Reading activeStandbyProperties");
 328         Properties activeStandbyProperties = new Properties();
 329         activeStandbyProperties.load(new FileInputStream(new File(
 330                 configDir + "/feature-active-standby-management.properties")));
 331         String thisPdpId = activeStandbyProperties
 332                 .getProperty(ActiveStandbyProperties.NODE_NAME);
 333
 334         logger.debug("testSanitizeDesignatedList: Creating emfDrools");
 335         EntityManagerFactory emfDrools = Persistence.createEntityManagerFactory(
 336                 "junitDroolsPU", activeStandbyProperties);
 337
 338         final DroolsPdpsConnector droolsPdpsConnector = new JpaDroolsPdpsConnector(emfDrools);
 339
 340         // Create 4 pdpd all not designated
 341
 342         DroolsPdp pdp1 = new DroolsPdpImpl("pdp1", false, 4, new Date());
 343         DroolsPdp pdp2 = new DroolsPdpImpl("pdp2", false, 4, new Date());
 344         DroolsPdp pdp3 = new DroolsPdpImpl("pdp3", false, 4, new Date());
 345         DroolsPdp pdp4 = new DroolsPdpImpl("pdp4", false, 4, new Date());
 346
 347         List<DroolsPdp> listOfDesignated = new ArrayList<DroolsPdp>();
 348         listOfDesignated.add(pdp1);
 349         listOfDesignated.add(pdp2);
 350         listOfDesignated.add(pdp3);
 351         listOfDesignated.add(pdp4);
 352
 353         // Now we want to create a StateManagementFeature and initialize it.  It will be
 354         // discovered by the ActiveStandbyFeature when the election handler initializes.
 355
 356         StateManagementFeatureApi stateManagementFeature = null;
 357         for (StateManagementFeatureApi feature : StateManagementFeatureApiConstants.getImpl().getList()) {
 358             ((PolicySessionFeatureApi) feature).globalInit(null, configDir);
 359             stateManagementFeature = feature;
 360             logger.debug("testColdStandby stateManagementFeature.getResourceName(): {}",
 361                     stateManagementFeature.getResourceName());
 362             break;
 363         }
 364         if (stateManagementFeature == null) {
 365             logger.error("testColdStandby failed to initialize.  "
 366                     + "Unable to get instance of StateManagementFeatureApi "
 367                     + "with resourceID: {}", thisPdpId);
 368             logger.debug("testColdStandby failed to initialize.  "
 369                     + "Unable to get instance of StateManagementFeatureApi "
 370                     + "with resourceID: {}", thisPdpId);
 371         }
 372
 373
 374         DroolsPdpsElectionHandler droolsPdpsElectionHandler =  new DroolsPdpsElectionHandler(droolsPdpsConnector, pdp1);
 375
 376         listOfDesignated = droolsPdpsElectionHandler.santizeDesignatedList(listOfDesignated);
 377
 378         logger.debug("\n\ntestSanitizeDesignatedList: listOfDesignated.size = {}\n\n",listOfDesignated.size());
 379
 380         assertTrue(listOfDesignated.size() == 4);
 381
 382         // Now make 2 designated
 383
 384         pdp1.setDesignated(true);
 385         pdp2.setDesignated(true);
 386
 387         listOfDesignated = droolsPdpsElectionHandler.santizeDesignatedList(listOfDesignated);
 388
 389         logger.debug("\n\ntestSanitizeDesignatedList: listOfDesignated.size after 2 designated = {}\n\n",
 390                 listOfDesignated.size());
 391
 392         assertTrue(listOfDesignated.size() == 2);
 393         assertTrue(listOfDesignated.contains(pdp1));
 394         assertTrue(listOfDesignated.contains(pdp2));
 395
 396
 397         // Now all are designated.  But, we have to add back the previously non-designated nodes
 398
 399         pdp3.setDesignated(true);
 400         pdp4.setDesignated(true);
 401         listOfDesignated.add(pdp3);
 402         listOfDesignated.add(pdp4);
 403
 404         listOfDesignated = droolsPdpsElectionHandler.santizeDesignatedList(listOfDesignated);
 405
 406         logger.debug("\n\ntestSanitizeDesignatedList: listOfDesignated.size after all designated = {}\n\n",
 407                 listOfDesignated.size());
 408
 409         assertTrue(listOfDesignated.size() == 4);
 410
 411     }
 412
 413     /**
 414     *  Test Compute most recent primary.
 415     *
 416     * @throws Exception exception
 417     */
 418     //@Ignore
 419     //@Test
 420     public void testComputeMostRecentPrimary() throws Exception {
 421
 422         logger.debug("\n\ntestComputeMostRecentPrimary: Entering\n\n");
 423
 424         logger.debug("testComputeMostRecentPrimary: Reading activeStandbyProperties");
 425         Properties activeStandbyProperties = new Properties();
 426         activeStandbyProperties.load(new FileInputStream(new File(
 427                 configDir + "/feature-active-standby-management.properties")));
 428         String thisPdpId = activeStandbyProperties
 429                 .getProperty(ActiveStandbyProperties.NODE_NAME);
 430
 431         logger.debug("testComputeMostRecentPrimary: Creating emfDrools");
 432         EntityManagerFactory emfDrools = Persistence.createEntityManagerFactory(
 433                 "junitDroolsPU", activeStandbyProperties);
 434
 435         final DroolsPdpsConnector droolsPdpsConnector = new JpaDroolsPdpsConnector(emfDrools);
 436
 437
 438         // Create 4 pdpd all not designated
 439
 440
 441         long designatedDateMs = new Date().getTime();
 442         DroolsPdp pdp1 = new DroolsPdpImpl("pdp1", false, 4, new Date());
 443         pdp1.setDesignatedDate(new Date(designatedDateMs - 2));
 444
 445         DroolsPdp pdp2 = new DroolsPdpImpl("pdp2", false, 4, new Date());
 446         //oldest
 447         pdp2.setDesignatedDate(new Date(designatedDateMs - 3));
 448
 449         DroolsPdp pdp3 = new DroolsPdpImpl("pdp3", false, 4, new Date());
 450         pdp3.setDesignatedDate(new Date(designatedDateMs - 1));
 451
 452         DroolsPdp pdp4 = new DroolsPdpImpl("pdp4", false, 4, new Date());
 453         //most recent
 454         pdp4.setDesignatedDate(new Date(designatedDateMs));
 455
 456         ArrayList<DroolsPdp> listOfAllPdps = new ArrayList<DroolsPdp>();
 457         listOfAllPdps.add(pdp1);
 458         listOfAllPdps.add(pdp2);
 459         listOfAllPdps.add(pdp3);
 460         listOfAllPdps.add(pdp4);
 461
 462
 463         ArrayList<DroolsPdp> listOfDesignated = new ArrayList<DroolsPdp>();
 464         listOfDesignated.add(pdp1);
 465         listOfDesignated.add(pdp2);
 466         listOfDesignated.add(pdp3);
 467         listOfDesignated.add(pdp4);
 468
 469         // Because the way we sanitize the listOfDesignated, it will always contain all hot standby
 470         // or all designated members.
 471
 472         // Now we want to create a StateManagementFeature and initialize it.  It will be
 473         // discovered by the ActiveStandbyFeature when the election handler initializes.
 474
 475         StateManagementFeatureApi stateManagementFeature = null;
 476         for (StateManagementFeatureApi feature : StateManagementFeatureApiConstants.getImpl().getList()) {
 477             ((PolicySessionFeatureApi) feature).globalInit(null, configDir);
 478             stateManagementFeature = feature;
 479             logger.debug("testComputeMostRecentPrimary stateManagementFeature.getResourceName(): {}",
 480                     stateManagementFeature.getResourceName());
 481             break;
 482         }
 483         if (stateManagementFeature == null) {
 484             logger.error("testComputeMostRecentPrimary failed to initialize.  "
 485                     + "Unable to get instance of StateManagementFeatureApi "
 486                     + "with resourceID: {}", thisPdpId);
 487             logger.debug("testComputeMostRecentPrimary failed to initialize.  "
 488                     + "Unable to get instance of StateManagementFeatureApi "
 489                     + "with resourceID: {}", thisPdpId);
 490         }
 491
 492         DroolsPdpsElectionHandler droolsPdpsElectionHandler =  new DroolsPdpsElectionHandler(droolsPdpsConnector, pdp1);
 493
 494         DroolsPdp mostRecentPrimary = droolsPdpsElectionHandler.computeMostRecentPrimary(
 495                 listOfAllPdps, listOfDesignated);
 496
 497         logger.debug("\n\ntestComputeMostRecentPrimary: mostRecentPrimary.getPdpId() = {}\n\n",
 498                 mostRecentPrimary.getPdpId());
 499
 500
 501         // If all of the pdps are included in the listOfDesignated and none are designated, it will choose
 502         // the one which has the most recent designated date.
 503
 504
 505         assertTrue(mostRecentPrimary.getPdpId().equals("pdp4"));
 506
 507
 508         // Now let's designate all of those on the listOfDesignated.  It will choose the first one designated
 509
 510
 511         pdp1.setDesignated(true);
 512         pdp2.setDesignated(true);
 513         pdp3.setDesignated(true);
 514         pdp4.setDesignated(true);
 515
 516         mostRecentPrimary = droolsPdpsElectionHandler.computeMostRecentPrimary(listOfAllPdps, listOfDesignated);
 517
 518         logger.debug("\n\ntestComputeMostRecentPrimary: All designated all on list, "
 519                 + "mostRecentPrimary.getPdpId() = {}\n\n",
 520                 mostRecentPrimary.getPdpId());
 521
 522
 523         // If all of the pdps are included in the listOfDesignated and all are designated, it will choose
 524         // the one which was designated first
 525
 526
 527         assertTrue(mostRecentPrimary.getPdpId().equals("pdp2"));
 528
 529
 530         // Now we will designate only 2 and put just them in the listOfDesignated.  The algorithm will now
 531         // look for the most recently designated pdp which is not currently designated.
 532
 533
 534         pdp3.setDesignated(false);
 535         pdp4.setDesignated(false);
 536
 537         listOfDesignated.remove(pdp3);
 538         listOfDesignated.remove(pdp4);
 539
 540         mostRecentPrimary = droolsPdpsElectionHandler.computeMostRecentPrimary(listOfAllPdps, listOfDesignated);
 541
 542         logger.debug("\n\ntestComputeMostRecentPrimary: mostRecentPrimary.getPdpId() = {}\n\n",
 543                 mostRecentPrimary.getPdpId());
 544
 545         assertTrue(mostRecentPrimary.getPdpId().equals("pdp4"));
 546
 547
 548
 549         // Now we will have none designated and put two of them in the listOfDesignated.  The algorithm will now
 550         // look for the most recently designated pdp regardless of whether it is currently marked as designated.
 551
 552
 553         pdp1.setDesignated(false);
 554         pdp2.setDesignated(false);
 555
 556         mostRecentPrimary = droolsPdpsElectionHandler.computeMostRecentPrimary(listOfAllPdps, listOfDesignated);
 557
 558         logger.debug("\n\ntestComputeMostRecentPrimary: 2 on list mostRecentPrimary.getPdpId() = {}\n\n",
 559                 mostRecentPrimary.getPdpId());
 560
 561         assertTrue(mostRecentPrimary.getPdpId().equals("pdp4"));
 562
 563
 564         // If we have only one pdp on in the listOfDesignated,
 565         // the most recently designated pdp will be chosen, regardless
 566         // of its designation status
 567
 568
 569         listOfDesignated.remove(pdp1);
 570
 571         mostRecentPrimary = droolsPdpsElectionHandler.computeMostRecentPrimary(listOfAllPdps, listOfDesignated);
 572
 573         logger.debug("\n\ntestComputeMostRecentPrimary: 1 on list mostRecentPrimary.getPdpId() = {}\n\n",
 574                 mostRecentPrimary.getPdpId());
 575
 576         assertTrue(mostRecentPrimary.getPdpId().equals("pdp4"));
 577
 578
 579         // Finally, if none are on the listOfDesignated, it will again choose the most recently designated pdp.
 580
 581
 582         listOfDesignated.remove(pdp2);
 583
 584         mostRecentPrimary = droolsPdpsElectionHandler.computeMostRecentPrimary(listOfAllPdps, listOfDesignated);
 585
 586         logger.debug("\n\ntestComputeMostRecentPrimary: 0 on list mostRecentPrimary.getPdpId() = {}\n\n",
 587                 mostRecentPrimary.getPdpId());
 588
 589         assertTrue(mostRecentPrimary.getPdpId().equals("pdp4"));
 590
 591     }
 592
 593     /**
 594      * Test compute designated PDP.
 595      *
 596      * @throws Exception exception
 597      */
 598     //@Ignore
 599     //@Test
 600     public void testComputeDesignatedPdp() throws Exception {
 601
 602         logger.debug("\n\ntestComputeDesignatedPdp: Entering\n\n");
 603
 604         logger.debug("testComputeDesignatedPdp: Reading activeStandbyProperties");
 605         Properties activeStandbyProperties = new Properties();
 606         activeStandbyProperties.load(new FileInputStream(new File(
 607                 configDir + "/feature-active-standby-management.properties")));
 608         String thisPdpId = activeStandbyProperties
 609                 .getProperty(ActiveStandbyProperties.NODE_NAME);
 610
 611
 612         logger.debug("testComputeDesignatedPdp: Creating emfDrools");
 613         EntityManagerFactory emfDrools = Persistence.createEntityManagerFactory(
 614                 "junitDroolsPU", activeStandbyProperties);
 615
 616         final DroolsPdpsConnector droolsPdpsConnector = new JpaDroolsPdpsConnector(emfDrools);
 617
 618
 619         // Create 4 pdpd all not designated.  Two on site1. Two on site2
 620
 621
 622         long designatedDateMs = new Date().getTime();
 623         DroolsPdp pdp1 = new DroolsPdpImpl("pdp1", false, 4, new Date());
 624         pdp1.setDesignatedDate(new Date(designatedDateMs - 2));
 625         pdp1.setSite("site1");
 626
 627         DroolsPdp pdp2 = new DroolsPdpImpl("pdp2", false, 4, new Date());
 628         pdp2.setDesignatedDate(new Date(designatedDateMs - 3));
 629         pdp2.setSite("site1");
 630
 631         //oldest
 632         DroolsPdp pdp3 = new DroolsPdpImpl("pdp3", false, 4, new Date());
 633         pdp3.setDesignatedDate(new Date(designatedDateMs - 4));
 634         pdp3.setSite("site2");
 635
 636         DroolsPdp pdp4 = new DroolsPdpImpl("pdp4", false, 4, new Date());
 637         //most recent
 638         pdp4.setDesignatedDate(new Date(designatedDateMs));
 639         pdp4.setSite("site2");
 640
 641         ArrayList<DroolsPdp> listOfAllPdps = new ArrayList<DroolsPdp>();
 642         listOfAllPdps.add(pdp1);
 643         listOfAllPdps.add(pdp2);
 644         listOfAllPdps.add(pdp3);
 645         listOfAllPdps.add(pdp4);
 646
 647
 648         ArrayList<DroolsPdp> listOfDesignated = new ArrayList<DroolsPdp>();
 649
 650
 651         // We will first test an empty listOfDesignated. As we know from the previous JUnit,
 652         // the pdp with the most designated date will be chosen for mostRecentPrimary
 653
 654         // Now we want to create a StateManagementFeature and initialize it.  It will be
 655         // discovered by the ActiveStandbyFeature when the election handler initializes.
 656
 657         StateManagementFeatureApi stateManagementFeature = null;
 658         for (StateManagementFeatureApi feature : StateManagementFeatureApiConstants.getImpl().getList()) {
 659             ((PolicySessionFeatureApi) feature).globalInit(null, configDir);
 660             stateManagementFeature = feature;
 661             logger.debug("testComputeDesignatedPdp stateManagementFeature.getResourceName(): {}",
 662                     stateManagementFeature.getResourceName());
 663             break;
 664         }
 665         if (stateManagementFeature == null) {
 666             logger.error("testComputeDesignatedPdp failed to initialize.  "
 667                     + "Unable to get instance of StateManagementFeatureApi "
 668                     + "with resourceID: {}", thisPdpId);
 669             logger.debug("testComputeDesignatedPdp failed to initialize.  "
 670                     + "Unable to get instance of StateManagementFeatureApi "
 671                     + "with resourceID: {}", thisPdpId);
 672         }
 673
 674
 675         DroolsPdpsElectionHandler droolsPdpsElectionHandler =  new DroolsPdpsElectionHandler(droolsPdpsConnector, pdp1);
 676
 677         DroolsPdp mostRecentPrimary = pdp4;
 678
 679         DroolsPdp designatedPdp = droolsPdpsElectionHandler.computeDesignatedPdp(listOfDesignated, mostRecentPrimary);
 680
 681
 682         // The designatedPdp should be null
 683
 684         assertTrue(designatedPdp == null);
 685
 686
 687         // Now let's try having only one pdp in listOfDesignated, but not in the same site as the most recent primary
 688
 689         listOfDesignated.add(pdp2);
 690
 691         designatedPdp = droolsPdpsElectionHandler.computeDesignatedPdp(listOfDesignated, mostRecentPrimary);
 692
 693
 694         // Now the designatedPdp should be the one and only selection in the listOfDesignated
 695
 696
 697         assertTrue(designatedPdp.getPdpId().equals(pdp2.getPdpId()));
 698
 699
 700         // Now let's put 2 pdps in the listOfDesignated, neither in the same site as the mostRecentPrimary
 701
 702
 703         listOfDesignated.add(pdp1);
 704
 705         designatedPdp = droolsPdpsElectionHandler.computeDesignatedPdp(listOfDesignated, mostRecentPrimary);
 706
 707
 708         // The designatedPdp should now be the one with the lowest lexiographic score - pdp1
 709
 710
 711         assertTrue(designatedPdp.getPdpId().equals(pdp1.getPdpId()));
 712
 713
 714         // Finally, we will have 2 pdps in the listOfDesignated, one in the same site with the mostRecentPrimary
 715
 716
 717         listOfDesignated.remove(pdp1);
 718         listOfDesignated.add(pdp3);
 719
 720         designatedPdp = droolsPdpsElectionHandler.computeDesignatedPdp(listOfDesignated, mostRecentPrimary);
 721
 722
 723         // The designatedPdp should now be the one on the same site as the mostRecentPrimary
 724
 725
 726         assertTrue(designatedPdp.getPdpId().equals(pdp3.getPdpId()));
 727     }
 728
 729     /**
 730      * Test cold standby.
 731      *
 732      * @throws Exception exception
 733      */
 734     //@Ignore
 735     //@Test
 736     public void testColdStandby() throws Exception {
 737
 738         logger.debug("\n\ntestColdStandby: Entering\n\n");
 739         cleanXacmlDb();
 740         cleanDroolsDb();
 741
 742         logger.debug("testColdStandby: Reading stateManagementProperties");
 743         Properties stateManagementProperties = new Properties();
 744         stateManagementProperties.load(new FileInputStream(new File(
 745                 configDir + "/feature-state-management.properties")));
 746
 747         logger.debug("testColdStandby: Creating emfXacml");
 748         final EntityManagerFactory emfXacml = Persistence.createEntityManagerFactory(
 749                 "junitXacmlPU", stateManagementProperties);
 750
 751         logger.debug("testColdStandby: Reading activeStandbyProperties");
 752         Properties activeStandbyProperties = new Properties();
 753         activeStandbyProperties.load(new FileInputStream(new File(
 754                 configDir + "/feature-active-standby-management.properties")));
 755         final String thisPdpId = activeStandbyProperties.getProperty(ActiveStandbyProperties.NODE_NAME);
 756
 757         logger.debug("testColdStandby: Creating emfDrools");
 758         EntityManagerFactory emfDrools = Persistence.createEntityManagerFactory(
 759                 "junitDroolsPU", activeStandbyProperties);
 760
 761         DroolsPdpsConnector conn = new JpaDroolsPdpsConnector(emfDrools);
 762
 763         logger.debug("testColdStandby: Cleaning up tables");
 764         conn.deleteAllPdps();
 765
 766         logger.debug("testColdStandby: Inserting PDP={} as designated", thisPdpId);
 767         DroolsPdp pdp = new DroolsPdpImpl(thisPdpId, true, 4, new Date());
 768         conn.insertPdp(pdp);
 769         DroolsPdpEntity droolsPdpEntity = conn.getPdp(thisPdpId);
 770         logger.debug("testColdStandby: After insertion, DESIGNATED= {} "
 771                 + "for PDP= {}", droolsPdpEntity.isDesignated(), thisPdpId);
 772         assertTrue(droolsPdpEntity.isDesignated() == true);
 773
 774         /*
 775          * When the Standby Status changes (from providingservice) to hotstandby
 776          * or coldstandby,the Active/Standby selection algorithm must stand down
 777          * if thePDP-D is currently the lead/active node and allow another PDP-D
 778          * to take over.
 779          *
 780          * It must also call lock on all engines in the engine management.
 781          */
 782
 783
 784         /*
 785          * Yes, this is kludgy, but we have a chicken and egg problem here: we
 786          * need a StateManagement object to invoke the
 787          * deleteAllStateManagementEntities method.
 788          */
 789         logger.debug("testColdStandby: Instantiating stateManagement object");
 790
 791         StateManagement sm = new StateManagement(emfXacml, "dummy");
 792         sm.deleteAllStateManagementEntities();
 793
 794         // Now we want to create a StateManagementFeature and initialize it.  It will be
 795         // discovered by the ActiveStandbyFeature when the election handler initializes.
 796
 797         StateManagementFeatureApi smf = null;
 798         for (StateManagementFeatureApi feature : StateManagementFeatureApiConstants.getImpl().getList()) {
 799             ((PolicySessionFeatureApi) feature).globalInit(null, configDir);
 800             smf = feature;
 801             logger.debug("testColdStandby stateManagementFeature.getResourceName(): {}", smf.getResourceName());
 802             break;
 803         }
 804         if (smf == null) {
 805             logger.error("testColdStandby failed to initialize.  "
 806                     + "Unable to get instance of StateManagementFeatureApi "
 807                     + "with resourceID: {}", thisPdpId);
 808             logger.debug("testColdStandby failed to initialize.  "
 809                     + "Unable to get instance of StateManagementFeatureApi "
 810                     + "with resourceID: {}", thisPdpId);
 811         }
 812
 813         // Create an ActiveStandbyFeature and initialize it. It will discover the StateManagementFeature
 814         // that has been created.
 815         ActiveStandbyFeatureApi activeStandbyFeature = null;
 816         for (ActiveStandbyFeatureApi feature : ActiveStandbyFeatureApiConstants.getImpl().getList()) {
 817             ((PolicySessionFeatureApi) feature).globalInit(null, configDir);
 818             activeStandbyFeature = feature;
 819             logger.debug("testColdStandby activeStandbyFeature.getResourceName(): {}",
 820                     activeStandbyFeature.getResourceName());
 821             break;
 822         }
 823         if (activeStandbyFeature == null) {
 824             logger.error("testColdStandby failed to initialize.  "
 825                     + "Unable to get instance of ActiveStandbyFeatureAPI "
 826                     + "with resourceID:{}", thisPdpId);
 827             logger.debug("testColdStandby failed to initialize.  "
 828                     + "Unable to get instance of ActiveStandbyFeatureAPI "
 829                     + "with resourceID:{}", thisPdpId);
 830         }
 831
 832         // Artificially putting a PDP into service is really a two step process, 1)
 833         // inserting it as designated and 2) promoting it so that its standbyStatus
 834         // is providing service.
 835
 836         logger.debug("testColdStandby: Runner started; Sleeping "
 837                 + interruptRecoveryTime + "ms before promoting PDP= {}",
 838                 thisPdpId);
 839         sleep(interruptRecoveryTime);
 840
 841         logger.debug("testColdStandby: Promoting PDP={}", thisPdpId);
 842         smf.promote();
 843
 844         String standbyStatus = sm.getStandbyStatus(thisPdpId);
 845         logger.debug("testColdStandby: Before locking, PDP= {}  has standbyStatus= {}",
 846                 thisPdpId, standbyStatus);
 847
 848         logger.debug("testColdStandby: Locking smf");
 849         smf.lock();
 850
 851         sleep(interruptRecoveryTime);
 852
 853         // Verify that the PDP is no longer designated.
 854
 855         droolsPdpEntity = conn.getPdp(thisPdpId);
 856         logger.debug("testColdStandby: After lock sm.lock() invoked, "
 857                 + "DESIGNATED= {} for PDP={}", droolsPdpEntity.isDesignated(), thisPdpId);
 858         assertTrue(droolsPdpEntity.isDesignated() == false);
 859
 860         logger.debug("\n\ntestColdStandby: Exiting\n\n");
 861         sleep(interruptRecoveryTime);
 862
 863     }
 864
 865     // Tests hot standby when there is only one PDP.
 866
 867     /**
 868      * Test hot standby 1.
 869      *
 870      * @throws Exception exception
 871      */
 872     //@Ignore
 873     //@Test
 874     public void testHotStandby1() throws Exception {
 875
 876         logger.debug("\n\ntestHotStandby1: Entering\n\n");
 877         cleanXacmlDb();
 878         cleanDroolsDb();
 879
 880         logger.debug("testHotStandby1: Reading stateManagementProperties");
 881         Properties stateManagementProperties = new Properties();
 882         stateManagementProperties.load(new FileInputStream(new File(
 883                 configDir + "/feature-state-management.properties")));
 884
 885         logger.debug("testHotStandby1: Creating emfXacml");
 886         final EntityManagerFactory emfXacml = Persistence.createEntityManagerFactory(
 887                 "junitXacmlPU", stateManagementProperties);
 888
 889         logger.debug("testHotStandby1: Reading activeStandbyProperties");
 890         Properties activeStandbyProperties = new Properties();
 891         activeStandbyProperties.load(new FileInputStream(new File(
 892                 configDir + "/feature-active-standby-management.properties")));
 893         final String thisPdpId = activeStandbyProperties
 894                 .getProperty(ActiveStandbyProperties.NODE_NAME);
 895
 896         logger.debug("testHotStandby1: Creating emfDrools");
 897         EntityManagerFactory emfDrools = Persistence.createEntityManagerFactory(
 898                 "junitDroolsPU", activeStandbyProperties);
 899
 900         DroolsPdpsConnector conn = new JpaDroolsPdpsConnector(emfDrools);
 901
 902         logger.debug("testHotStandby1: Cleaning up tables");
 903         conn.deleteAllPdps();
 904
 905         /*
 906          * Insert this PDP as not designated.  Initial standby state will be
 907          * either null or cold standby.   Demoting should transit state to
 908          * hot standby.
 909          */
 910
 911         logger.debug("testHotStandby1: Inserting PDP={} as not designated", thisPdpId);
 912         Date yesterday = DateUtils.addDays(new Date(), -1);
 913         DroolsPdpImpl pdp = new DroolsPdpImpl(thisPdpId, false, 4, yesterday);
 914         conn.insertPdp(pdp);
 915         DroolsPdpEntity droolsPdpEntity = conn.getPdp(thisPdpId);
 916         logger.debug("testHotStandby1: After insertion, PDP={} has DESIGNATED={}",
 917                 thisPdpId, droolsPdpEntity.isDesignated());
 918         assertTrue(droolsPdpEntity.isDesignated() == false);
 919
 920         logger.debug("testHotStandby1: Instantiating stateManagement object");
 921         StateManagement sm = new StateManagement(emfXacml, "dummy");
 922         sm.deleteAllStateManagementEntities();
 923
 924
 925         // Now we want to create a StateManagementFeature and initialize it.  It will be
 926         // discovered by the ActiveStandbyFeature when the election handler initializes.
 927
 928         StateManagementFeatureApi smf = null;
 929         for (StateManagementFeatureApi feature : StateManagementFeatureApiConstants.getImpl().getList()) {
 930             ((PolicySessionFeatureApi) feature).globalInit(null, configDir);
 931             smf = feature;
 932             logger.debug("testHotStandby1 stateManagementFeature.getResourceName(): {}", smf.getResourceName());
 933             break;
 934         }
 935         if (smf == null) {
 936             logger.error("testHotStandby1 failed to initialize.  "
 937                     + "Unable to get instance of StateManagementFeatureApi "
 938                     + "with resourceID: {}", thisPdpId);
 939             logger.debug("testHotStandby1 failed to initialize.  "
 940                     + "Unable to get instance of StateManagementFeatureApi "
 941                     + "with resourceID: {}", thisPdpId);
 942         }
 943
 944         // Create an ActiveStandbyFeature and initialize it. It will discover the StateManagementFeature
 945         // that has been created.
 946         ActiveStandbyFeatureApi activeStandbyFeature = null;
 947         for (ActiveStandbyFeatureApi feature : ActiveStandbyFeatureApiConstants.getImpl().getList()) {
 948             ((PolicySessionFeatureApi) feature).globalInit(null, configDir);
 949             activeStandbyFeature = feature;
 950             logger.debug("testHotStandby1 activeStandbyFeature.getResourceName(): {}",
 951                     activeStandbyFeature.getResourceName());
 952             break;
 953         }
 954         if (activeStandbyFeature == null) {
 955             logger.error("testHotStandby1 failed to initialize.  "
 956                     + "Unable to get instance of ActiveStandbyFeatureAPI "
 957                     + "with resourceID: {}", thisPdpId);
 958             logger.debug("testHotStandby1 failed to initialize.  "
 959                     + "Unable to get instance of ActiveStandbyFeatureAPI "
 960                     + "with resourceID: {}", thisPdpId);
 961         }
 962
 963
 964         logger.debug("testHotStandby1: Demoting PDP={}", thisPdpId);
 965         // demoting should cause state to transit to hotstandby
 966         smf.demote();
 967
 968
 969         logger.debug("testHotStandby1: Sleeping {} ms, to allow JpaDroolsPdpsConnector "
 970                 + "time to check droolspdpentity table", sleepTime);
 971         sleep(sleepTime);
 972
 973
 974         // Verify that this formerly un-designated PDP in HOT_STANDBY is now designated and providing service.
 975
 976         droolsPdpEntity = conn.getPdp(thisPdpId);
 977         logger.debug("testHotStandby1: After sm.demote() invoked, DESIGNATED= {} "
 978                 + "for PDP= {}", droolsPdpEntity.isDesignated(), thisPdpId);
 979         assertTrue(droolsPdpEntity.isDesignated() == true);
 980         String standbyStatus = smf.getStandbyStatus(thisPdpId);
 981         logger.debug("testHotStandby1: After demotion, PDP= {} "
 982                 + "has standbyStatus= {}", thisPdpId, standbyStatus);
 983         assertTrue(standbyStatus != null  &&  standbyStatus.equals(StateManagement.PROVIDING_SERVICE));
 984
 985         logger.debug("testHotStandby1: Stopping policyManagementRunner");
 986
 987         logger.debug("\n\ntestHotStandby1: Exiting\n\n");
 988         sleep(interruptRecoveryTime);
 989
 990     }
 991
 992     /*
 993      * Tests hot standby when two PDPs are involved.
 994      */
 995
 996     /**
 997      * Test hot standby 2.
 998      *
 999      * @throws Exception exception
1000      */
1001     //@Ignore
1002     //@Test
1003     public void testHotStandby2() throws Exception {
1004
1005         logger.info("\n\ntestHotStandby2: Entering\n\n");
1006         cleanXacmlDb();
1007         cleanDroolsDb();
1008
1009         logger.info("testHotStandby2: Reading stateManagementProperties");
1010         Properties stateManagementProperties = new Properties();
1011         stateManagementProperties.load(new FileInputStream(new File(
1012                 configDir + "/feature-state-management.properties")));
1013
1014         logger.info("testHotStandby2: Creating emfXacml");
1015         final EntityManagerFactory emfXacml = Persistence.createEntityManagerFactory(
1016                 "junitXacmlPU", stateManagementProperties);
1017
1018         logger.info("testHotStandby2: Reading activeStandbyProperties");
1019         Properties activeStandbyProperties = new Properties();
1020         activeStandbyProperties.load(new FileInputStream(new File(
1021                 configDir + "/feature-active-standby-management.properties")));
1022         final String thisPdpId = activeStandbyProperties
1023                 .getProperty(ActiveStandbyProperties.NODE_NAME);
1024
1025         logger.info("testHotStandby2: Creating emfDrools");
1026         EntityManagerFactory emfDrools = Persistence.createEntityManagerFactory(
1027                 "junitDroolsPU", activeStandbyProperties);
1028
1029         DroolsPdpsConnector conn = new JpaDroolsPdpsConnector(emfDrools);
1030
1031         logger.info("testHotStandby2: Cleaning up tables");
1032         conn.deleteAllPdps();
1033
1034
1035         // Insert a PDP that's designated but not current.
1036
1037         String activePdpId = "pdp2";
1038         logger.info("testHotStandby2: Inserting PDP={} as stale, designated PDP", activePdpId);
1039         Date yesterday = DateUtils.addDays(new Date(), -1);
1040         DroolsPdp pdp = new DroolsPdpImpl(activePdpId, true, 4, yesterday);
1041         conn.insertPdp(pdp);
1042         DroolsPdpEntity droolsPdpEntity = conn.getPdp(activePdpId);
1043         logger.info("testHotStandby2: After insertion, PDP= {}, which is "
1044                 + "not current, has DESIGNATED= {}", activePdpId, droolsPdpEntity.isDesignated());
1045         assertTrue(droolsPdpEntity.isDesignated() == true);
1046
1047         /*
1048          * Promote the designated PDP.
1049          *
1050          * We have a chicken and egg problem here: we need a StateManagement
1051          * object to invoke the deleteAllStateManagementEntities method.
1052          */
1053
1054
1055         logger.info("testHotStandby2: Promoting PDP={}", activePdpId);
1056         StateManagement sm = new StateManagement(emfXacml, "dummy");
1057         sm.deleteAllStateManagementEntities();
1058
1059
1060         sm = new StateManagement(emfXacml, activePdpId);//pdp2
1061
1062         // Artificially putting a PDP into service is really a two step process, 1)
1063         // inserting it as designated and 2) promoting it so that its standbyStatus
1064         // is providing service.
1065
1066         /*
1067          * Insert this PDP as not designated.  Initial standby state will be
1068          * either null or cold standby.   Demoting should transit state to
1069          * hot standby.
1070          */
1071
1072
1073         logger.info("testHotStandby2: Inserting PDP= {} as not designated", thisPdpId);
1074         pdp = new DroolsPdpImpl(thisPdpId, false, 4, yesterday);
1075         conn.insertPdp(pdp);
1076         droolsPdpEntity = conn.getPdp(thisPdpId);
1077         logger.info("testHotStandby2: After insertion, PDP={} "
1078                 + "has DESIGNATED= {}", thisPdpId, droolsPdpEntity.isDesignated());
1079         assertTrue(droolsPdpEntity.isDesignated() == false);
1080
1081
1082         // Now we want to create a StateManagementFeature and initialize it.  It will be
1083         // discovered by the ActiveStandbyFeature when the election handler initializes.
1084
1085         StateManagementFeatureApi sm2 = null;
1086         for (StateManagementFeatureApi feature : StateManagementFeatureApiConstants.getImpl().getList()) {
1087             ((PolicySessionFeatureApi) feature).globalInit(null, configDir);
1088             sm2 = feature;
1089             logger.debug("testHotStandby2 stateManagementFeature.getResourceName(): {}", sm2.getResourceName());
1090             break;
1091         }
1092         if (sm2 == null) {
1093             logger.error("testHotStandby2 failed to initialize.  "
1094                     + "Unable to get instance of StateManagementFeatureApi "
1095                     + "with resourceID: {}", thisPdpId);
1096             logger.debug("testHotStandby2 failed to initialize.  "
1097                     + "Unable to get instance of StateManagementFeatureApi "
1098                     + "with resourceID: {}", thisPdpId);
1099         }
1100
1101         // Create an ActiveStandbyFeature and initialize it. It will discover the StateManagementFeature
1102         // that has been created.
1103         ActiveStandbyFeatureApi activeStandbyFeature = null;
1104         for (ActiveStandbyFeatureApi feature : ActiveStandbyFeatureApiConstants.getImpl().getList()) {
1105             ((PolicySessionFeatureApi) feature).globalInit(null, configDir);
1106             activeStandbyFeature = feature;
1107             logger.debug("testHotStandby2 activeStandbyFeature.getResourceName(): {}",
1108                     activeStandbyFeature.getResourceName());
1109             break;
1110         }
1111         if (activeStandbyFeature == null) {
1112             logger.error("testHotStandby2 failed to initialize.  "
1113                     + "Unable to get instance of ActiveStandbyFeatureAPI "
1114                     + "with resourceID: {}", thisPdpId);
1115             logger.debug("testHotStandby2 failed to initialize.  "
1116                     + "Unable to get instance of ActiveStandbyFeatureAPI "
1117                     + "with resourceID: {}", thisPdpId);
1118         }
1119
1120         logger.info("testHotStandby2: Runner started; Sleeping {} "
1121                 + "ms before promoting/demoting", interruptRecoveryTime);
1122         sleep(interruptRecoveryTime);
1123
1124         logger.info("testHotStandby2: Runner started; promoting PDP={}", activePdpId);
1125         //At this point, the newly created pdp will have set the state to disabled/failed/cold standby
1126         //because it is stale. So, it cannot be promoted.  We need to call sm.enableNotFailed() so we
1127         //can promote it and demote the other pdp - else the other pdp will just spring back to providingservice
1128         sm.enableNotFailed();//pdp2
1129         sm.promote();
1130         String standbyStatus = sm.getStandbyStatus(activePdpId);
1131         logger.info("testHotStandby2: After promoting, PDP= {} has standbyStatus= {}", activePdpId, standbyStatus);
1132
1133         // demoting PDP should ensure that state transits to hotstandby
1134         logger.info("testHotStandby2: Runner started; demoting PDP= {}", thisPdpId);
1135         sm2.demote();//pdp1
1136         standbyStatus = sm.getStandbyStatus(thisPdpId);
1137         logger.info("testHotStandby2: After demoting, PDP={} has standbyStatus= {}",thisPdpId , standbyStatus);
1138
1139         logger.info("testHotStandby2: Sleeping {} ms, to allow JpaDroolsPdpsConnector "
1140                 + "time to check droolspdpentity table", sleepTime);
1141         sleep(sleepTime);
1142
1143         /*
1144          * Verify that this PDP, demoted to HOT_STANDBY, is now
1145          * re-designated and providing service.
1146          */
1147
1148         droolsPdpEntity = conn.getPdp(thisPdpId);
1149         logger.info("testHotStandby2: After demoting PDP={}"
1150                 + ", DESIGNATED= {}"
1151                 + " for PDP= {}", activePdpId, droolsPdpEntity.isDesignated(), thisPdpId);
1152         assertTrue(droolsPdpEntity.isDesignated() == true);
1153         standbyStatus = sm2.getStandbyStatus(thisPdpId);
1154         logger.info("testHotStandby2: After demoting PDP={}"
1155                 + ", PDP={} has standbyStatus= {}",
1156                 activePdpId, thisPdpId, standbyStatus);
1157         assertTrue(standbyStatus != null
1158                 && standbyStatus.equals(StateManagement.PROVIDING_SERVICE));
1159
1160         logger.info("testHotStandby2: Stopping policyManagementRunner");
1161
1162         logger.info("\n\ntestHotStandby2: Exiting\n\n");
1163         sleep(interruptRecoveryTime);
1164
1165     }
1166
1167     /*
1168      * 1) Inserts and designates this PDP, then verifies that startTransaction
1169      * is successful.
1170      *
1171      * 2) Demotes PDP, and verifies that because there is only one PDP, it will
1172      * be immediately re-promoted, thus allowing startTransaction to be
1173      * successful.
1174      *
1175      * 3) Locks PDP and verifies that startTransaction results in
1176      * AdministrativeStateException.
1177      *
1178      * 4) Unlocks PDP and verifies that startTransaction results in
1179      * StandbyStatusException.
1180      *
1181      * 5) Promotes PDP and verifies that startTransaction is once again
1182      * successful.
1183      */
1184
1185     /**
1186      * Test locking.
1187      *
1188      * @throws Exception exception
1189      */
1190     //@Ignore
1191     //@Test
1192     public void testLocking1() throws Exception {
1193         logger.debug("testLocking1: Entry");
1194         cleanXacmlDb();
1195         cleanDroolsDb();
1196
1197         logger.debug("testLocking1: Reading stateManagementProperties");
1198         Properties stateManagementProperties = new Properties();
1199         stateManagementProperties.load(new FileInputStream(new File(
1200                 configDir + "/feature-state-management.properties")));
1201
1202         logger.debug("testLocking1: Creating emfXacml");
1203         final EntityManagerFactory emfXacml = Persistence.createEntityManagerFactory(
1204                 "junitXacmlPU", stateManagementProperties);
1205
1206         logger.debug("testLocking1: Reading activeStandbyProperties");
1207         Properties activeStandbyProperties = new Properties();
1208         activeStandbyProperties.load(new FileInputStream(new File(
1209                 configDir + "/feature-active-standby-management.properties")));
1210         final String thisPdpId = activeStandbyProperties
1211                 .getProperty(ActiveStandbyProperties.NODE_NAME);
1212
1213         logger.debug("testLocking1: Creating emfDrools");
1214         EntityManagerFactory emfDrools = Persistence.createEntityManagerFactory(
1215                 "junitDroolsPU", activeStandbyProperties);
1216
1217         DroolsPdpsConnector conn = new JpaDroolsPdpsConnector(emfDrools);
1218
1219         logger.debug("testLocking1: Cleaning up tables");
1220         conn.deleteAllPdps();
1221
1222         /*
1223          * Insert this PDP as designated.  Initial standby state will be
1224          * either null or cold standby.
1225          */
1226
1227         logger.debug("testLocking1: Inserting PDP= {} as designated", thisPdpId);
1228         DroolsPdpImpl pdp = new DroolsPdpImpl(thisPdpId, true, 4, new Date());
1229         conn.insertPdp(pdp);
1230         DroolsPdpEntity droolsPdpEntity = conn.getPdp(thisPdpId);
1231         logger.debug("testLocking1: After insertion, PDP= {} has DESIGNATED= {}",
1232                 thisPdpId, droolsPdpEntity.isDesignated());
1233         assertTrue(droolsPdpEntity.isDesignated() == true);
1234
1235         logger.debug("testLocking1: Instantiating stateManagement object");
1236         StateManagement smDummy = new StateManagement(emfXacml, "dummy");
1237         smDummy.deleteAllStateManagementEntities();
1238
1239         // Now we want to create a StateManagementFeature and initialize it.  It will be
1240         // discovered by the ActiveStandbyFeature when the election handler initializes.
1241
1242         StateManagementFeatureApi sm = null;
1243         for (StateManagementFeatureApi feature : StateManagementFeatureApiConstants.getImpl().getList()) {
1244             ((PolicySessionFeatureApi) feature).globalInit(null, configDir);
1245             sm = feature;
1246             logger.debug("testLocking1 stateManagementFeature.getResourceName(): {}", sm.getResourceName());
1247             break;
1248         }
1249         if (sm == null) {
1250             logger.error("testLocking1 failed to initialize.  "
1251                     + "Unable to get instance of StateManagementFeatureApi "
1252                     + "with resourceID: {}", thisPdpId);
1253             logger.debug("testLocking1 failed to initialize.  "
1254                     + "Unable to get instance of StateManagementFeatureApi "
1255                     + "with resourceID: {}", thisPdpId);
1256         }
1257
1258         // Create an ActiveStandbyFeature and initialize it. It will discover the StateManagementFeature
1259         // that has been created.
1260         ActiveStandbyFeatureApi activeStandbyFeature = null;
1261         for (ActiveStandbyFeatureApi feature : ActiveStandbyFeatureApiConstants.getImpl().getList()) {
1262             ((PolicySessionFeatureApi) feature).globalInit(null, configDir);
1263             activeStandbyFeature = feature;
1264             logger.debug("testLocking1 activeStandbyFeature.getResourceName(): {}",
1265                     activeStandbyFeature.getResourceName());
1266             break;
1267         }
1268         if (activeStandbyFeature == null) {
1269             logger.error("testLocking1 failed to initialize.  "
1270                     + "Unable to get instance of ActiveStandbyFeatureAPI "
1271                     + "with resourceID: {}", thisPdpId);
1272             logger.debug("testLocking1 failed to initialize.  "
1273                     + "Unable to get instance of ActiveStandbyFeatureAPI "
1274                     + "with resourceID: {}", thisPdpId);
1275         }
1276
1277         logger.debug("testLocking1: Runner started; Sleeping "
1278                 + interruptRecoveryTime + "ms before promoting PDP={}",
1279                 thisPdpId);
1280         sleep(interruptRecoveryTime);
1281
1282         logger.debug("testLocking1: Promoting PDP={}", thisPdpId);
1283         sm.promote();
1284
1285         logger.debug("testLocking1: Sleeping {} ms, to allow time for "
1286                 + "policy-management.Main class to come up, designated= {}",
1287                 sleepTime, conn.getPdp(thisPdpId).isDesignated());
1288         sleep(sleepTime);
1289
1290         logger.debug("testLocking1: Waking up and invoking startTransaction on active PDP={}"
1291                 + ", designated= {}",thisPdpId, conn.getPdp(thisPdpId).isDesignated());
1292
1293
1294         IntegrityMonitor droolsPdpIntegrityMonitor = IntegrityMonitor.getInstance();
1295         try {
1296             droolsPdpIntegrityMonitor.startTransaction();
1297             droolsPdpIntegrityMonitor.endTransaction();
1298             logger.debug("testLocking1: As expected, transaction successful");
1299         } catch (AdministrativeStateException e) {
1300             logger.error("testLocking1: Unexpectedly caught AdministrativeStateException, ", e);
1301             assertTrue(false);
1302         } catch (StandbyStatusException e) {
1303             logger.error("testLocking1: Unexpectedly caught StandbyStatusException, ", e);
1304             assertTrue(false);
1305         } catch (Exception e) {
1306             logger.error("testLocking1: Unexpectedly caught Exception, ", e);
1307             assertTrue(false);
1308         }
1309
1310         // demoting should cause state to transit to hotstandby, followed by re-promotion,
1311         // since there is only one PDP.
1312         logger.debug("testLocking1: demoting PDP={}", thisPdpId);
1313         sm.demote();
1314
1315         logger.debug("testLocking1: sleeping" + electionWaitSleepTime
1316                 + " to allow election handler to re-promote PDP={}", thisPdpId);
1317         sleep(electionWaitSleepTime);
1318
1319         logger.debug("testLocking1: Invoking startTransaction on re-promoted PDP={}"
1320                 + ", designated={}", thisPdpId, conn.getPdp(thisPdpId).isDesignated());
1321         try {
1322             droolsPdpIntegrityMonitor.startTransaction();
1323             droolsPdpIntegrityMonitor.endTransaction();
1324             logger.debug("testLocking1: As expected, transaction successful");
1325         } catch (AdministrativeStateException e) {
1326             logger.error("testLocking1: Unexpectedly caught AdministrativeStateException, ", e);
1327             assertTrue(false);
1328         } catch (StandbyStatusException e) {
1329             logger.error("testLocking1: Unexpectedly caught StandbyStatusException, ", e);
1330             assertTrue(false);
1331         } catch (Exception e) {
1332             logger.error("testLocking1: Unexpectedly caught Exception, ", e);
1333             assertTrue(false);
1334         }
1335
1336         // locking should cause state to transit to cold standby
1337         logger.debug("testLocking1: locking PDP={}", thisPdpId);
1338         sm.lock();
1339
1340         // Just to avoid any race conditions, sleep a little after locking
1341         logger.debug("testLocking1: Sleeping a few millis after locking, to avoid race condition");
1342         sleep(100);
1343
1344         logger.debug("testLocking1: Invoking startTransaction on locked PDP= {}"
1345                 + ", designated= {}",thisPdpId, conn.getPdp(thisPdpId).isDesignated());
1346         try {
1347             droolsPdpIntegrityMonitor.startTransaction();
1348             logger.error("testLocking1: startTransaction unexpectedly successful");
1349             assertTrue(false);
1350         } catch (AdministrativeStateException e) {
1351             logger.debug("testLocking1: As expected, caught AdministrativeStateException, ", e);
1352         } catch (StandbyStatusException e) {
1353             logger.error("testLocking1: Unexpectedly caught StandbyStatusException, ", e);
1354             assertTrue(false);
1355         } catch (Exception e) {
1356             logger.error("testLocking1: Unexpectedly caught Exception, ", e);
1357             assertTrue(false);
1358         } finally {
1359             droolsPdpIntegrityMonitor.endTransaction();
1360         }
1361
1362         // unlocking should cause state to transit to hot standby and then providing service
1363         logger.debug("testLocking1: unlocking PDP={}", thisPdpId);
1364         sm.unlock();
1365
1366         // Just to avoid any race conditions, sleep a little after locking
1367         logger.debug("testLocking1: Sleeping a few millis after unlocking, to avoid race condition");
1368         sleep(electionWaitSleepTime);
1369
1370         logger.debug("testLocking1: Invoking startTransaction on unlocked PDP="
1371                 + thisPdpId
1372                 + ", designated="
1373                 + conn.getPdp(thisPdpId).isDesignated());
1374         try {
1375             droolsPdpIntegrityMonitor.startTransaction();
1376             logger.error("testLocking1: startTransaction successful as expected");
1377         } catch (AdministrativeStateException e) {
1378             logger.error("testLocking1: Unexpectedly caught AdministrativeStateException, ", e);
1379             assertTrue(false);
1380         } catch (StandbyStatusException e) {
1381             logger.debug("testLocking1: Unexpectedly caught StandbyStatusException, ", e);
1382             assertTrue(false);
1383         } catch (Exception e) {
1384             logger.error("testLocking1: Unexpectedly caught Exception, ", e);
1385             assertTrue(false);
1386         } finally {
1387             droolsPdpIntegrityMonitor.endTransaction();
1388         }
1389
1390         // demoting should cause state to transit to hot standby
1391         logger.debug("testLocking1: demoting PDP={}", thisPdpId);
1392         sm.demote();
1393
1394         // Just to avoid any race conditions, sleep a little after promoting
1395         logger.debug("testLocking1: Sleeping a few millis after demoting, to avoid race condition");
1396         sleep(100);
1397
1398         logger.debug("testLocking1: Invoking startTransaction on demoted PDP={}"
1399                 + ", designated={}", thisPdpId, conn.getPdp(thisPdpId).isDesignated());
1400         try {
1401             droolsPdpIntegrityMonitor.startTransaction();
1402             droolsPdpIntegrityMonitor.endTransaction();
1403             logger.debug("testLocking1: Unexpectedly, transaction successful");
1404             assertTrue(false);
1405         } catch (AdministrativeStateException e) {
1406             logger.error("testLocking1: Unexpectedly caught AdministrativeStateException, ", e);
1407             assertTrue(false);
1408         } catch (StandbyStatusException e) {
1409             logger.error("testLocking1: As expected caught StandbyStatusException, ", e);
1410         } catch (Exception e) {
1411             logger.error("testLocking1: Unexpectedly caught Exception, ", e);
1412             assertTrue(false);
1413         }
1414
1415         logger.debug("\n\ntestLocking1: Exiting\n\n");
1416         sleep(interruptRecoveryTime);
1417
1418     }
1419
1420
1421     /*
1422      * 1) Inserts and designates this PDP, then verifies that startTransaction
1423      * is successful.
1424      *
1425      * 2) Inserts another PDP in hotstandby.
1426      *
1427      * 3) Demotes this PDP, and verifies 1) that other PDP is not promoted (because one
1428      * PDP cannot promote another PDP) and 2) that this PDP is re-promoted.
1429      */
1430
1431     /**
1432      * Test locking 2.
1433      *
1434      * @throws Exception exception
1435      */
1436     //@Ignore
1437     //@Test
1438     public void testLocking2() throws Exception {
1439
1440         logger.debug("\n\ntestLocking2: Entering\n\n");
1441         cleanXacmlDb();
1442         cleanDroolsDb();
1443
1444         logger.debug("testLocking2: Reading stateManagementProperties");
1445         Properties stateManagementProperties = new Properties();
1446         stateManagementProperties.load(new FileInputStream(new File(
1447                 configDir + "/feature-state-management.properties")));
1448
1449         logger.debug("testLocking2: Creating emfXacml");
1450         final EntityManagerFactory emfXacml = Persistence.createEntityManagerFactory(
1451                 "junitXacmlPU", stateManagementProperties);
1452
1453         logger.debug("testLocking2: Reading activeStandbyProperties");
1454         Properties activeStandbyProperties = new Properties();
1455         activeStandbyProperties.load(new FileInputStream(new File(
1456                 configDir + "/feature-active-standby-management.properties")));
1457         final String thisPdpId = activeStandbyProperties
1458                 .getProperty(ActiveStandbyProperties.NODE_NAME);
1459
1460         logger.debug("testLocking2: Creating emfDrools");
1461         EntityManagerFactory emfDrools = Persistence.createEntityManagerFactory(
1462                 "junitDroolsPU", activeStandbyProperties);
1463
1464         DroolsPdpsConnector conn = new JpaDroolsPdpsConnector(emfDrools);
1465
1466         logger.debug("testLocking2: Cleaning up tables");
1467         conn.deleteAllPdps();
1468
1469         /*
1470          * Insert this PDP as designated.  Initial standby state will be
1471          * either null or cold standby.   Demoting should transit state to
1472          * hot standby.
1473          */
1474
1475         logger.debug("testLocking2: Inserting PDP= {} as designated", thisPdpId);
1476         DroolsPdpImpl pdp = new DroolsPdpImpl(thisPdpId, true, 3, new Date());
1477         conn.insertPdp(pdp);
1478         DroolsPdpEntity droolsPdpEntity = conn.getPdp(thisPdpId);
1479         logger.debug("testLocking2: After insertion, PDP= {} has DESIGNATED= {}",
1480                 thisPdpId, droolsPdpEntity.isDesignated());
1481         assertTrue(droolsPdpEntity.isDesignated() == true);
1482
1483         logger.debug("testLocking2: Instantiating stateManagement object and promoting PDP={}", thisPdpId);
1484         StateManagement smDummy = new StateManagement(emfXacml, "dummy");
1485         smDummy.deleteAllStateManagementEntities();
1486
1487         // Now we want to create a StateManagementFeature and initialize it.  It will be
1488         // discovered by the ActiveStandbyFeature when the election handler initializes.
1489
1490         StateManagementFeatureApi sm = null;
1491         for (StateManagementFeatureApi feature : StateManagementFeatureApiConstants.getImpl().getList()) {
1492             ((PolicySessionFeatureApi) feature).globalInit(null, configDir);
1493             sm = feature;
1494             logger.debug("testLocking2 stateManagementFeature.getResourceName(): {}", sm.getResourceName());
1495             break;
1496         }
1497         if (sm == null) {
1498             logger.error("testLocking2 failed to initialize.  "
1499                     + "Unable to get instance of StateManagementFeatureApi "
1500                     + "with resourceID: {}", thisPdpId);
1501             logger.debug("testLocking2 failed to initialize.  "
1502                     + "Unable to get instance of StateManagementFeatureApi "
1503                     + "with resourceID: {}", thisPdpId);
1504         }
1505
1506         // Create an ActiveStandbyFeature and initialize it. It will discover the StateManagementFeature
1507         // that has been created.
1508         ActiveStandbyFeatureApi activeStandbyFeature = null;
1509         for (ActiveStandbyFeatureApi feature : ActiveStandbyFeatureApiConstants.getImpl().getList()) {
1510             ((PolicySessionFeatureApi) feature).globalInit(null, configDir);
1511             activeStandbyFeature = feature;
1512             logger.debug("testLocking2 activeStandbyFeature.getResourceName(): {}",
1513                     activeStandbyFeature.getResourceName());
1514             break;
1515         }
1516         if (activeStandbyFeature == null) {
1517             logger.error("testLocking2 failed to initialize.  "
1518                     + "Unable to get instance of ActiveStandbyFeatureAPI "
1519                     + "with resourceID: {}", thisPdpId);
1520             logger.debug("testLocking2 failed to initialize.  "
1521                     + "Unable to get instance of ActiveStandbyFeatureAPI "
1522                     + "with resourceID: {}", thisPdpId);
1523         }
1524
1525         /*
1526          * Insert another PDP as not designated.  Initial standby state will be
1527          * either null or cold standby.   Demoting should transit state to
1528          * hot standby.
1529          */
1530
1531         String standbyPdpId = "pdp2";
1532         logger.debug("testLocking2: Inserting PDP= {} as not designated", standbyPdpId);
1533         Date yesterday = DateUtils.addDays(new Date(), -1);
1534         pdp = new DroolsPdpImpl(standbyPdpId, false, 4, yesterday);
1535         conn.insertPdp(pdp);
1536         droolsPdpEntity = conn.getPdp(standbyPdpId);
1537         logger.debug("testLocking2: After insertion, PDP={} has DESIGNATED= {}",
1538                 standbyPdpId, droolsPdpEntity.isDesignated());
1539         assertTrue(droolsPdpEntity.isDesignated() == false);
1540
1541         logger.debug("testLocking2: Demoting PDP= {}", standbyPdpId);
1542         final StateManagement sm2 = new StateManagement(emfXacml, standbyPdpId);
1543
1544         logger.debug("testLocking2: Runner started; Sleeping {} ms "
1545                 + "before promoting/demoting", interruptRecoveryTime);
1546         sleep(interruptRecoveryTime);
1547
1548         logger.debug("testLocking2: Promoting PDP= {}", thisPdpId);
1549         sm.promote();
1550
1551         // demoting PDP should ensure that state transits to hotstandby
1552         logger.debug("testLocking2: Demoting PDP={}", standbyPdpId);
1553         sm2.demote();
1554
1555         logger.debug("testLocking2: Sleeping {} ms, to allow time for to come up", sleepTime);
1556         sleep(sleepTime);
1557
1558         logger.debug("testLocking2: Waking up and invoking startTransaction on active PDP={}"
1559                 + ", designated= {}", thisPdpId, conn.getPdp(thisPdpId).isDesignated());
1560
1561         IntegrityMonitor droolsPdpIntegrityMonitor = IntegrityMonitor.getInstance();
1562
1563         try {
1564             droolsPdpIntegrityMonitor.startTransaction();
1565             droolsPdpIntegrityMonitor.endTransaction();
1566             logger.debug("testLocking2: As expected, transaction successful");
1567         } catch (AdministrativeStateException e) {
1568             logger.error("testLocking2: Unexpectedly caught AdministrativeStateException, ", e);
1569             assertTrue(false);
1570         } catch (StandbyStatusException e) {
1571             logger.error("testLocking2: Unexpectedly caught StandbyStatusException, ", e);
1572             assertTrue(false);
1573         } catch (Exception e) {
1574             logger.error("testLocking2: Unexpectedly caught Exception, ", e);
1575             assertTrue(false);
1576         }
1577
1578         // demoting should cause state to transit to hotstandby followed by re-promotion.
1579         logger.debug("testLocking2: demoting PDP={}", thisPdpId);
1580         sm.demote();
1581
1582         logger.debug("testLocking2: sleeping {}"
1583                 + " to allow election handler to re-promote PDP={}", electionWaitSleepTime, thisPdpId);
1584         sleep(electionWaitSleepTime);
1585
1586         logger.debug("testLocking2: Waking up and invoking startTransaction "
1587                 + "on re-promoted PDP= {}, designated= {}",
1588                 thisPdpId, conn.getPdp(thisPdpId).isDesignated());
1589         try {
1590             droolsPdpIntegrityMonitor.startTransaction();
1591             droolsPdpIntegrityMonitor.endTransaction();
1592             logger.debug("testLocking2: As expected, transaction successful");
1593         } catch (AdministrativeStateException e) {
1594             logger.error("testLocking2: Unexpectedly caught AdministrativeStateException, ", e);
1595             assertTrue(false);
1596         } catch (StandbyStatusException e) {
1597             logger.error("testLocking2: Unexpectedly caught StandbyStatusException, ", e);
1598             assertTrue(false);
1599         } catch (Exception e) {
1600             logger.error("testLocking2: Unexpectedly caught Exception, ", e);
1601             assertTrue(false);
1602         }
1603
1604         logger.debug("testLocking2: Verifying designated status for PDP= {}", standbyPdpId);
1605         boolean standbyPdpDesignated = conn.getPdp(standbyPdpId).isDesignated();
1606         assertTrue(standbyPdpDesignated == false);
1607
1608         logger.debug("\n\ntestLocking2: Exiting\n\n");
1609         sleep(interruptRecoveryTime);
1610     }
1611
1612     private void sleep(long sleepms) throws InterruptedException {
1613         Thread.sleep(sleepms);
1614     }
1615 }