Batch submit 03/4003/1
authorRalph Straubs <rs8887@att.com>
Tue, 9 May 2017 08:17:21 +0000 (03:17 -0500)
committerRalph Straubs <rs8887@att.com>
Tue, 9 May 2017 08:20:15 +0000 (03:20 -0500)
[ECOMPD2TD-000] Update version.properties for iter 41

[US866186 1707] Adding methods to IntegrityMonitor to support state
audit.

[US866186 1710] Made more progress on teh stateAudit method of
IntegritryMontor.  Completed all except the addition of properties.

[US866186 1710] Addition of
IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS and logic to
stateAudit to determine whether or not to run

[US866186 1710] Created a JUnit for getAllForwardProgressEntity()
method and debugged and fixed it.

[US866186 1710] Making progress on the testStateAudit() JUnit.  Was able
to create forward progress entries and manually change the lastUpdated
date on an entry using a persistence update query.

[US866186 1710] First cut of stateAudit JUnit test.  Still need to test
more cases, but I got it working so far.

[US866186 1710] Completed the JUnit for stateAudit.  Added  5 new
scenarios.

Conflicts:

version.properties

Change-Id: I7b5622a27021cf01b5d916b75824d8e51b5252eb
Signed-off-by: Ralph Straubs <rs8887@att.com>
integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitor.java
integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitorProperties.java
integrity-monitor/src/test/java/org/openecomp/policy/common/im/test/IntegrityMonitorTest.java

index 293bd11..a421894 100644 (file)
@@ -21,6 +21,7 @@
 package org.openecomp.policy.common.im;
 
 import java.net.InetAddress;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
@@ -101,6 +102,14 @@ public class IntegrityMonitor {
        // This allows time for dependents to come up.
        private long lastDependencyCheckTime = System.currentTimeMillis();
        
+       // Time of the last state audit.  It is initialized at the time of the IM construction
+       private Date lastStateAuditTime = new Date();
+       
+       //Interval between state audits in ms.  We leave it turned off by default so that it will only
+       //be run on the nodes which we want doing the audit. In particular, we only want it to run
+       //on the droolspdps
+       private static long stateAuditIntervalMs = 0L;
+       
        // the number of cycles since 'fpCounter' was last changed
        private int missedCycles = 0;
        
@@ -647,6 +656,53 @@ public class IntegrityMonitor {
                return error_msg;
        }
        
+       public ArrayList<ForwardProgressEntity> getAllForwardProgressEntity(){
+               if(logger.isDebugEnabled()){
+                       logger.debug("getAllForwardProgressEntity: entry");
+               }
+               ArrayList<ForwardProgressEntity> fpList = new ArrayList<ForwardProgressEntity>();
+               // Start a transaction
+               EntityTransaction et = em.getTransaction();
+               et.begin();
+               try {
+                       Query fquery = em.createQuery("Select e from ForwardProgressEntity e");
+                       @SuppressWarnings("rawtypes")
+                       List myList = fquery.setLockMode(
+                                         LockModeType.NONE).setFlushMode(FlushModeType.COMMIT).getResultList();
+                       synchronized(IMFLUSHLOCK){
+                               et.commit();
+                       }
+                       if(logger.isDebugEnabled()){
+                               logger.debug("getAllForwardProgressEntity: myList.size(): " + myList.size());
+                       }
+                       if(!myList.isEmpty()){
+                               for(int i = 0; i < myList.size(); i++){
+                                       if(logger.isDebugEnabled()){
+                                               logger.debug("getAllForwardProgressEntity: myList.get(" + i +").getResourceName()" 
+                                                       + ": " + ((ForwardProgressEntity)myList.get(i)).getResourceName());
+                                       }
+                                       fpList.add((ForwardProgressEntity) myList.get(i));
+                               }
+                       }
+                       synchronized(IMFLUSHLOCK){
+                               if(et.isActive()){
+                                       et.commit();
+                               }
+                       }
+               } catch (Exception e) {
+                       // log an error and continue
+                       String msg = "getAllForwardProgessEntity DB read failed with exception: " + e;
+                       logger.error(msg);
+                       synchronized(IMFLUSHLOCK){
+                               if(et.isActive()){
+                                       et.rollback();
+                               }
+                       }
+               }
+               return fpList;
+       }
+         
+       
        private String jmxCheck(String dep) {
                logger.debug("checking health of dependent by calling test() via JMX on resource: " + dep);
 
@@ -1117,6 +1173,14 @@ public class IntegrityMonitor {
                        }
                }
                
+               if (prop.getProperty(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS) != null){
+                       try{
+                               stateAuditIntervalMs = Long.parseLong(prop.getProperty(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS));
+                       }catch(NumberFormatException e){
+                               logger.warn("Ignored invalid property: " + IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS);
+                       }
+               }
+               
                
                return;
        }
@@ -1203,6 +1267,111 @@ public class IntegrityMonitor {
                }
        }
        
+       /**
+        * Look for "Forward Progress" on other nodes.  If they are not making forward progress,
+        * check their operational state.  If it is not disabled, then disable them.
+        */
+       public void stateAudit() {
+               
+               //TODO add stateAuditIntervalMs to the IntegrityMonitor properties here and in droolspdp
+               // monitoring interval checks
+               if (stateAuditIntervalMs <= 0) {
+                       return; // stateAudit is disabled
+               }
+               
+               //Only run from nodes that are operational
+               if(stateManager.getOpState().equals(StateManagement.DISABLED)){
+                       return;
+               }
+               if(stateManager.getAdminState().equals(StateManagement.LOCKED)){
+                       return;
+               }
+               if(!stateManager.getStandbyStatus().equals(StateManagement.NULL_VALUE) &&
+                               stateManager.getStandbyStatus()!= null){
+                       if(!stateManager.getStandbyStatus().equals(StateManagement.PROVIDING_SERVICE)){
+                               return;
+                       }
+               }
+
+               Date date = new Date();         
+               long timeSinceLastStateAudit = date.getTime() - lastStateAuditTime.getTime(); 
+               if (timeSinceLastStateAudit < stateAuditIntervalMs){
+                       return;
+               }
+
+               // Get all entries in the forwardprogressentity table
+               ArrayList<ForwardProgressEntity> fpList = getAllForwardProgressEntity();
+
+               // Check if each forwardprogressentity entry is current
+               for(ForwardProgressEntity fpe : fpList){
+                       //If the this is my ForwardProgressEntity, continue
+                       if(fpe.getResourceName().equals(IntegrityMonitor.resourceName)){
+                               continue;
+                       }
+                       long diffMs = date.getTime() - fpe.getLastUpdated().getTime();
+                       logger.debug("IntegrityMonitor.stateAudit(): diffMs = " + diffMs);
+
+                       //Threshold for a stale entry
+                       long staleMs = failedCounterThreshold * monitorInterval * 1000;
+                       logger.debug("IntegrityMonitor.stateAudit(): staleMs = " + staleMs);
+
+                       if(diffMs > staleMs){
+                               //ForwardProgress is stale.  Disable it
+                               // Start a transaction
+                               EntityTransaction et = em.getTransaction();
+                               et.begin();
+                               StateManagementEntity sme = null;
+                               try {
+                                       // query if StateManagement entry exists for fpe resource
+                                       Query query = em.createQuery("Select p from StateManagementEntity p where p.resourceName=:resource");
+                                       query.setParameter("resource", fpe.getResourceName());
+
+                                       @SuppressWarnings("rawtypes")
+                                       List smList = query.setLockMode(
+                                                       LockModeType.NONE).setFlushMode(FlushModeType.COMMIT).getResultList();
+                                       if (!smList.isEmpty()) {
+                                               // exists
+                                               sme = (StateManagementEntity) smList.get(0);
+                                               // refresh the object from DB in case cached data was returned
+                                               em.refresh(sme);
+                                               logger.debug("IntegrityMonitor.stateAudit(): Found entry in StateManagementEntity table for Resource=" + sme.getResourceName());
+                                       } else {
+                                               String msg = "IntegrityMonitor.stateAudit(): " + fpe.getResourceName() + ": resource not found in state management entity database table";
+                                               logger.debug(msg);
+                                               logger.error(msg);
+                                       }
+                                       synchronized(IMFLUSHLOCK){
+                                               et.commit();
+                                       }
+                               } catch (Exception e) {
+                                       // log an error
+                                       String msg = "IntegrityMonitor.stateAudit(): " + fpe.getResourceName() + ": StateManagementEntity DB read failed with exception: " + e;
+                                       logger.debug(msg);
+                                       logger.error(msg);
+                                       synchronized(IMFLUSHLOCK){
+                                               if(et.isActive()){
+                                                       et.rollback();
+                                               }
+                                       }
+                               }
+
+                               if(sme != null){
+                                       if(!sme.getOpState().equals(StateManagement.DISABLED)){
+                                               logger.debug("IntegrityMonitor.stateAudit(): Changing OpStat = disabled for " + sme.getResourceName());
+                                               try {
+                                                       stateManager.disableFailed(sme.getResourceName());
+                                               } catch (Exception e) {
+                                                       String msg = "IntegrityMonitor.stateAudit(): Failed to disable " + sme.getResourceName();
+                                                       logger.debug(msg);
+                                                       logger.error(msg);
+                                               }
+                                       }
+                               }
+                       }// end if(diffMs > staleMs)
+               }// end for(ForwardProgressEntity fpe : fpList)
+               lastStateAuditTime = date;
+       }// end stateAudit()
+
        /**
         * Execute a test transaction when test transaction interval has elapsed.
         */
@@ -1319,7 +1488,7 @@ public class IntegrityMonitor {
        
        /**
         * The following nested class periodically performs the forward progress check,
-        * checks dependencies and does a refresh state audit.
+        * checks dependencies, does a refresh state audit and runs the stateAudit.
         */
        class FPManager extends Thread {
                
@@ -1372,6 +1541,12 @@ public class IntegrityMonitor {
                                        // check if it is time to run the refreshStateAudit
                                        IntegrityMonitor.this.refreshStateAudit();
                                        
+                                       if(logger.isDebugEnabled()){
+                                               logger.debug("FPManager calling stateAudit()");
+                                       }
+                                       // check if it is time to run the stateAudit
+                                       IntegrityMonitor.this.stateAudit();
+                                       
                                } catch (Exception e) {
                                        logger.debug("Ignore FPManager thread processing timer(s) exception: " + e);
                                }
index 8c3d7e6..9b9ae55 100644 (file)
@@ -51,5 +51,6 @@ public class IntegrityMonitorProperties {
        public static final String TEST_VIA_JMX = "test_via_jmx";
        public static final String JMX_FQDN = "jmx_fqdn";
        public static final String MAX_FPC_UPDATE_INTERVAL = "max_fpc_update_interval";
+       public static final String STATE_AUDIT_INTERVAL_MS = "state_audit_interval_ms";
        
 }
index 7375988..acc9ad0 100644 (file)
@@ -22,6 +22,7 @@ package org.openecomp.policy.common.im.test;
 
 import static org.junit.Assert.*;
 
+import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 import java.util.Properties;
@@ -31,6 +32,7 @@ import javax.persistence.EntityManagerFactory;
 import javax.persistence.EntityTransaction;
 import javax.persistence.Persistence;
 import javax.persistence.Query;
+import javax.persistence.TemporalType;
 
 import org.junit.After;
 import org.junit.AfterClass;
@@ -117,11 +119,13 @@ public class IntegrityMonitorTest {
        @Ignore
        @Test
        public void runAllTests() throws Exception{
-               testSanityJmx();
-               testIM();
-               testSanityState();
-               testRefreshStateAudit();
-               testStateCheck();
+               //testSanityJmx();
+               //testIM();
+               //testSanityState();
+               //testRefreshStateAudit();
+               //testStateCheck();
+               //testGetAllForwardProgressEntity();
+               testStateAudit();
        }
 
        /*
@@ -833,4 +837,398 @@ public class IntegrityMonitorTest {
                System.out.println("\n\ntestStateCheck: Exit\n\n");
        }
        
+       public void testGetAllForwardProgressEntity() throws Exception{
+               System.out.println("\nIntegrityMonitorTest: Entering testGetAllForwardProgressEntity\n\n");
+               logger.debug("\nIntegrityMonitorTest: Entering testGetAllForwardProgressEntity\n\n");
+               
+               // parameters are passed via a properties file
+               myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "");
+               IntegrityMonitor.updateProperties(myProp);
+               
+               et = em.getTransaction();
+               et.begin();
+
+               // Make sure we start with the DB clean
+               em.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
+               em.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
+               em.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
+
+               em.flush();
+               et.commit();
+
+               
+               IntegrityMonitor.deleteInstance();
+               IntegrityMonitor im = IntegrityMonitor.getInstance(resourceName, myProp);
+               
+               logger.debug("\nIntegrityMonitorTest: Creating ForwardProgressEntity entries\n\n");
+               // Add a resources to put an entry in the forward progress table
+               ForwardProgressEntity fpe = new ForwardProgressEntity();
+               ForwardProgressEntity fpe2 = new ForwardProgressEntity();
+               ForwardProgressEntity fpe3 = new ForwardProgressEntity();
+               fpe.setFpcCount(0);
+               fpe.setResourceName("siteA_pap2");
+               fpe2.setFpcCount(0);
+               fpe2.setResourceName("siteB_pap1");
+               fpe3.setFpcCount(0);
+               fpe3.setResourceName("siteB_pap2");
+               et = em.getTransaction();
+               et.begin();
+               em.persist(fpe);
+               em.persist(fpe2);
+               em.persist(fpe3);
+               em.flush();
+               et.commit();
+
+               logger.debug("\nIntegrityMonitorTest:testGetAllForwardProgressEntity Calling im.getAllForwardProgressEntity()\n\n");
+               ArrayList<ForwardProgressEntity> fpeList = im.getAllForwardProgressEntity();
+               
+               assertTrue(fpeList.size()==4);
+               
+               et = em.getTransaction();
+               
+               et.begin();
+               // Make sure we leave the DB clean
+               em.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
+               em.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
+               em.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
+
+               em.flush();
+               et.commit();
+               
+               logger.debug("\nIntegrityMonitorTest: Exit testGetAllForwardProgressEntity\n\n");
+               System.out.println("\n\ntestGetAllForwardProgressEntity: Exit\n\n");
+       }
+       
+       public void testStateAudit() throws Exception{
+               System.out.println("\nIntegrityMonitorTest: Entering testStateAudit\n\n");
+               logger.debug("\nIntegrityMonitorTest: Entering testStateAudit\n\n");
+               
+               // parameters are passed via a properties file
+               myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "");
+               myProp.put(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS, "100");
+               IntegrityMonitor.updateProperties(myProp);
+               
+               et = em.getTransaction();
+               et.begin();
+
+               // Make sure we start with the DB clean
+               em.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
+               em.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
+               em.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
+
+               em.flush();
+               et.commit();
+
+               
+               IntegrityMonitor.deleteInstance();
+               IntegrityMonitor im = IntegrityMonitor.getInstance(resourceName, myProp);
+               
+               logger.debug("\nIntegrityMonitorTest: Creating ForwardProgressEntity entries\n\n");
+               // Add resources to put an entry in the forward progress table
+               Date staleDate = new Date(0);
+               ForwardProgressEntity fpe1 = new ForwardProgressEntity();
+               ForwardProgressEntity fpe2 = new ForwardProgressEntity();
+               ForwardProgressEntity fpe3 = new ForwardProgressEntity();
+               fpe1.setFpcCount(0);
+               fpe1.setResourceName("siteA_pap2");
+               fpe2.setFpcCount(0);
+               fpe2.setResourceName("siteB_pap1");
+               fpe3.setFpcCount(0);
+               fpe3.setResourceName("siteB_pap2");
+               logger.debug("\nIntegrityMonitorTest: Creating StateManagementEntity entries\n\n");
+               StateManagementEntity sme1 = new StateManagementEntity();
+               StateManagementEntity sme2 = new StateManagementEntity();
+               StateManagementEntity sme3= new StateManagementEntity();
+               sme1.setResourceName("siteA_pap2");
+               sme1.setAdminState(StateManagement.UNLOCKED);
+               sme1.setOpState(StateManagement.ENABLED);
+               sme1.setAvailStatus(StateManagement.NULL_VALUE);
+               sme1.setStandbyStatus(StateManagement.NULL_VALUE);
+               sme2.setResourceName("siteB_pap1");
+               sme2.setAdminState(StateManagement.UNLOCKED);
+               sme2.setOpState(StateManagement.ENABLED);
+               sme2.setAvailStatus(StateManagement.NULL_VALUE);
+               sme2.setStandbyStatus(StateManagement.NULL_VALUE);
+               sme3.setResourceName("siteB_pap2");
+               sme3.setAdminState(StateManagement.UNLOCKED);
+               sme3.setOpState(StateManagement.ENABLED);
+               sme3.setAvailStatus(StateManagement.NULL_VALUE);
+               sme3.setStandbyStatus(StateManagement.NULL_VALUE);
+               et = em.getTransaction();
+               et.begin();
+               em.persist(fpe1);
+               em.persist(fpe2);
+               em.persist(fpe3);
+               em.persist(sme1);
+               em.persist(sme2);
+               em.persist(sme3);
+               em.flush();
+               et.commit();
+               
+               Query updateQuery = em.createQuery("UPDATE ForwardProgressEntity f "
+                               + "SET f.lastUpdated = :newDate "
+                               + "WHERE f.resourceName=:resource");
+               updateQuery.setParameter("newDate", staleDate, TemporalType.TIMESTAMP);
+               updateQuery.setParameter("resource", fpe1.getResourceName());
+               
+               et = em.getTransaction();
+               et.begin();
+               updateQuery.executeUpdate();
+               et.commit();
+               
+               logger.debug("\nIntegrityMonitorTest:testStateAudit Calling im.getAllForwardProgressEntity()\n\n");
+               ArrayList<ForwardProgressEntity> fpeList = im.getAllForwardProgressEntity();
+               
+               logger.debug("\n\n");
+               logger.debug("IntegrityMonitorTest:testStateAudit:ForwardProgressEntity entries");
+               for(ForwardProgressEntity myFpe : fpeList){
+                       logger.debug("\n    ResourceName: " + myFpe.getResourceName()
+                                       + "\n        LastUpdated: " + myFpe.getLastUpdated());
+               }
+               logger.debug("\n\n");
+               
+               logger.debug("\nIntegrityMonitorTest:testStateAudit getting list of StateManagementEntity entries\n\n");
+               Query query = em.createQuery("SELECT s FROM StateManagementEntity s");
+               List smeList = query.getResultList();
+               
+               logger.debug("\n\n");
+               logger.debug("IntegrityMonitorTest:testStateAudit:StateManagementEntity entries");
+               for(Object mySme : smeList){
+                       StateManagementEntity tmpSme = (StateManagementEntity) mySme;
+                       em.refresh(tmpSme);
+                       logger.debug("\n    ResourceName: " + tmpSme.getResourceName()
+                                       + "\n        AdminState: " + tmpSme.getAdminState()
+                                       + "\n        OpState: " + tmpSme.getOpState()
+                                       + "\n        AvailStatus: " + tmpSme.getAvailStatus()
+                                       + "\n        StandbyStatus: " + tmpSme.getStandbyStatus()
+                                       );
+               }
+               logger.debug("\n\n");
+               
+               logger.debug("IntegrityMonitorTest:testStateAudit: sleeping 2 sec");
+               Thread.sleep(3000);
+               logger.debug("IntegrityMonitorTest:testStateAudit: Awake!");
+               
+               logger.debug("\nIntegrityMonitorTest:testStateAudit getting list of StateManagementEntity entries\n\n");
+               smeList = query.getResultList();
+               
+               logger.debug("\n\n");
+               logger.debug("IntegrityMonitorTest:testStateAudit:StateManagementEntity entries");
+               for(Object mySme : smeList){
+                       StateManagementEntity tmpSme = (StateManagementEntity) mySme;
+                       em.refresh(tmpSme);
+                       logger.debug("\n    ResourceName: " + tmpSme.getResourceName()
+                                       + "\n        AdminState: " + tmpSme.getAdminState()
+                                       + "\n        OpState: " + tmpSme.getOpState()
+                                       + "\n        AvailStatus: " + tmpSme.getAvailStatus()
+                                       + "\n        StandbyStatus: " + tmpSme.getStandbyStatus()
+                                       );
+               }
+               logger.debug("\n\n");
+               
+               em.refresh(sme1);
+               assertTrue(sme1.getOpState().equals(StateManagement.DISABLED));
+               
+               
+               //Now lock this IM
+               StateManagement sm = im.getStateManager();
+               sm.lock();
+               
+               //Give it time to write the db
+               Thread.sleep(2000);
+
+               //Put things back to their starting condition           
+               et = em.getTransaction();
+               et.begin();
+               sme1.setOpState(StateManagement.ENABLED);
+               sme1.setAvailStatus(StateManagement.NULL_VALUE);
+               em.persist(sme1);
+               et.commit();
+
+               //Now it should not update sme1
+               logger.debug("IntegrityMonitorTest:testStateAudit: 2nd sleeping 2 sec");
+               Thread.sleep(2000);
+               logger.debug("IntegrityMonitorTest:testStateAudit: 2nd Awake!");
+               
+               logger.debug("\nIntegrityMonitorTest:testStateAudit 2nd getting list of StateManagementEntity entries\n\n");
+               smeList = query.getResultList();
+               
+               logger.debug("\n\n");
+               logger.debug("IntegrityMonitorTest:testStateAudit:StateManagementEntity 2nd entries");
+               for(Object mySme : smeList){
+                       StateManagementEntity tmpSme = (StateManagementEntity) mySme;
+                       em.refresh(tmpSme);
+                       logger.debug("\n    2nd ResourceName: " + tmpSme.getResourceName()
+                                       + "\n        AdminState: " + tmpSme.getAdminState()
+                                       + "\n        OpState: " + tmpSme.getOpState()
+                                       + "\n        AvailStatus: " + tmpSme.getAvailStatus()
+                                       + "\n        StandbyStatus: " + tmpSme.getStandbyStatus()
+                                       );
+               }
+               logger.debug("\n\n");
+               
+               em.refresh(sme1);
+               assertTrue(sme1.getOpState().equals(StateManagement.ENABLED));
+               
+               //Now create a reason for this IM to be disabled.  Add a bogus dependency
+               myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "Bogus_Node");
+               IntegrityMonitor.updateProperties(myProp);
+
+               //Restart the IM
+               IntegrityMonitor.deleteInstance();
+               im = IntegrityMonitor.getInstance(resourceName, myProp);
+               
+               //Give it time to initialize and check dependencies
+               logger.debug("IntegrityMonitorTest:testStateAudit: (restart) sleeping 10 sec");
+               Thread.sleep(7000);
+               logger.debug("IntegrityMonitorTest:testStateAudit: (restart) Awake!");
+               
+               //Now unlock this IM.  Now it should be unlocked, but disabled due to dependency
+               sm.unlock();
+               
+               //Now check its state
+               logger.debug("\nIntegrityMonitorTest:testStateAudit (restart) getting list of StateManagementEntity entries\n\n");
+               smeList = query.getResultList();
+               
+               logger.debug("\n\n");
+               logger.debug("IntegrityMonitorTest:testStateAudit:StateManagementEntity (restart) entries");
+               for(Object mySme : smeList){
+                       StateManagementEntity tmpSme = (StateManagementEntity) mySme;
+                       em.refresh(tmpSme);
+                       logger.debug("\n    (restart) ResourceName: " + tmpSme.getResourceName()
+                                       + "\n        AdminState: " + tmpSme.getAdminState()
+                                       + "\n        OpState: " + tmpSme.getOpState()
+                                       + "\n        AvailStatus: " + tmpSme.getAvailStatus()
+                                       + "\n        StandbyStatus: " + tmpSme.getStandbyStatus()
+                                       );
+               }
+               logger.debug("\n\n");
+               
+               em.refresh(sme1);
+               assertTrue(sme1.getOpState().equals(StateManagement.ENABLED));
+               
+               //Now lock this IM so it will not audit when it comes back up
+               sm.lock();
+               
+               //Remove the bogus dependency and restart it
+               myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "");
+               IntegrityMonitor.updateProperties(myProp);
+
+               //Restart the IM
+               IntegrityMonitor.deleteInstance();
+               im = IntegrityMonitor.getInstance(resourceName, myProp);
+
+               //Give it time to initialize and check dependencies
+               logger.debug("IntegrityMonitorTest:testStateAudit: (restart2) sleeping 10 sec");
+               Thread.sleep(7000);
+               logger.debug("IntegrityMonitorTest:testStateAudit: (restart2) Awake!");
+               
+               //Now check its state
+               logger.debug("\nIntegrityMonitorTest:testStateAudit (restart2) getting list of StateManagementEntity entries\n\n");
+               smeList = query.getResultList();
+               
+               logger.debug("\n\n");
+               logger.debug("IntegrityMonitorTest:testStateAudit:StateManagementEntity (restart2) entries");
+               for(Object mySme : smeList){
+                       StateManagementEntity tmpSme = (StateManagementEntity) mySme;
+                       em.refresh(tmpSme);
+                       logger.debug("\n    (restart2) ResourceName: " + tmpSme.getResourceName()
+                                       + "\n        AdminState: " + tmpSme.getAdminState()
+                                       + "\n        OpState: " + tmpSme.getOpState()
+                                       + "\n        AvailStatus: " + tmpSme.getAvailStatus()
+                                       + "\n        StandbyStatus: " + tmpSme.getStandbyStatus()
+                                       );
+               }
+               logger.debug("\n\n");
+               
+               em.refresh(sme1);
+               assertTrue(sme1.getOpState().equals(StateManagement.ENABLED));
+               
+               //Make this IM coldstandby
+               sm.demote();
+               //Give it time to write the DB
+               Thread.sleep(2000);
+               //unlock it
+               sm.unlock();
+               //Give it time to write the DB
+               Thread.sleep(2000);
+               
+               //Now check its state
+               logger.debug("\nIntegrityMonitorTest:testStateAudit (restart3) getting list of StateManagementEntity entries\n\n");
+               smeList = query.getResultList();
+               
+               logger.debug("\n\n");
+               logger.debug("IntegrityMonitorTest:testStateAudit:StateManagementEntity (restart3) entries");
+               for(Object mySme : smeList){
+                       StateManagementEntity tmpSme = (StateManagementEntity) mySme;
+                       em.refresh(tmpSme);
+                       logger.debug("\n    (restart3) ResourceName: " + tmpSme.getResourceName()
+                                       + "\n        AdminState: " + tmpSme.getAdminState()
+                                       + "\n        OpState: " + tmpSme.getOpState()
+                                       + "\n        AvailStatus: " + tmpSme.getAvailStatus()
+                                       + "\n        StandbyStatus: " + tmpSme.getStandbyStatus()
+                                       );
+               }
+               logger.debug("\n\n");
+               
+               //sme1 should not be changed because this IM is hotstandby and cannot change its state
+               em.refresh(sme1);
+               assertTrue(sme1.getOpState().equals(StateManagement.ENABLED));
+               
+               //Now let's add sme2 to the mix
+               updateQuery = em.createQuery("UPDATE ForwardProgressEntity f "
+                               + "SET f.lastUpdated = :newDate "
+                               + "WHERE f.resourceName=:resource");
+               updateQuery.setParameter("newDate", staleDate, TemporalType.TIMESTAMP);
+               updateQuery.setParameter("resource", fpe2.getResourceName());
+               
+               et = em.getTransaction();
+               et.begin();
+               updateQuery.executeUpdate();
+               et.commit();
+               
+               //Finally, we want to promote this IM so it will disable sme1
+               sm.promote();
+               //Give it a chance to write the DB and run the audit
+               logger.debug("IntegrityMonitorTest:testStateAudit: (restart4) sleeping 2 sec");
+               Thread.sleep(3000);
+               logger.debug("IntegrityMonitorTest:testStateAudit: (restart4) Awake!");
+               
+               //Now check its state
+               logger.debug("\nIntegrityMonitorTest:testStateAudit (restart4) getting list of StateManagementEntity entries\n\n");
+               smeList = query.getResultList();
+               
+               logger.debug("\n\n");
+               logger.debug("IntegrityMonitorTest:testStateAudit:StateManagementEntity (restart4) entries");
+               for(Object mySme : smeList){
+                       StateManagementEntity tmpSme = (StateManagementEntity) mySme;
+                       em.refresh(tmpSme);
+                       logger.debug("\n    (restart4) ResourceName: " + tmpSme.getResourceName()
+                                       + "\n        AdminState: " + tmpSme.getAdminState()
+                                       + "\n        OpState: " + tmpSme.getOpState()
+                                       + "\n        AvailStatus: " + tmpSme.getAvailStatus()
+                                       + "\n        StandbyStatus: " + tmpSme.getStandbyStatus()
+                                       );
+               }
+               logger.debug("\n\n");
+               
+               em.refresh(sme1);
+               assertTrue(sme1.getOpState().equals(StateManagement.DISABLED));
+               
+               em.refresh(sme2);
+               assertTrue(sme2.getOpState().equals(StateManagement.DISABLED));
+               
+               et = em.getTransaction();
+               et.begin();
+               // Make sure we leave the DB clean
+               em.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
+               em.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
+               em.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
+
+               em.flush();
+               et.commit();
+               
+               logger.debug("\nIntegrityMonitorTest: Exit testStateAudit\n\n");
+               System.out.println("\n\ntestStateAudit: Exit\n\n");
+       }
 }