Changes to handle PDPX deploy/undeploy 01/85101/5
authorMichael Mokry <michael.mokry@att.com>
Thu, 11 Apr 2019 17:19:50 +0000 (12:19 -0500)
committerMichael Mokry <michael.mokry@att.com>
Thu, 11 Apr 2019 22:15:07 +0000 (17:15 -0500)
Added changes to handle incoming udpate for deploying or undeploying the
policies by comparing both sets.  STill need to add support for removing
the policies and getting the deployed ToscaPolicyIdentifiers for
PdpStatus response.

* PLD - added functionality to unload policies in the applications and
* PLD - added map of loaded policies

* Mike - made change to address Jim's and Joshua's comments
* Mike - Made change to use mapLoadedPolicies to get deployed policies
from the applications
* Mike - made change to assembly.xml to add support for apps directory.

Change-Id: If7d119197171b68b905b7fd0f2b5be6cf6f609e7
Issue-ID: POLICY-1451
Signed-off-by: Michael Mokry <michael.mokry@att.com>
Signed-off-by: Pamela Dragosh <pdragosh@research.att.com>
applications/common/src/main/java/org/onap/policy/pdp/xacml/application/common/TestUtils.java
applications/common/src/main/java/org/onap/policy/pdp/xacml/application/common/XacmlApplicationServiceProvider.java
applications/common/src/main/java/org/onap/policy/pdp/xacml/application/common/XacmlPolicyUtils.java
applications/common/src/main/java/org/onap/policy/pdp/xacml/application/common/std/StdXacmlApplicationServiceProvider.java
applications/common/src/test/java/org/onap/policy/pdp/xacml/application/common/XacmlPolicyUtilsTest.java
applications/monitoring/src/test/java/org/onap/policy/xacml/pdp/application/monitoring/MonitoringPdpApplicationTest.java
main/src/main/java/org/onap/policy/pdpx/main/comm/XacmlPdpMessage.java
main/src/main/java/org/onap/policy/pdpx/main/comm/XacmlPdpUpdatePublisher.java
main/src/main/java/org/onap/policy/pdpx/main/rest/XacmlPdpApplicationManager.java
packages/policy-xacmlpdp-tarball/src/main/package/tarball/assembly.xml

index fa32516..50eb50b 100644 (file)
@@ -22,6 +22,8 @@
 
 package org.onap.policy.pdp.xacml.application.common;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 
@@ -30,9 +32,12 @@ import org.onap.policy.common.utils.coder.StandardCoder;
 import org.onap.policy.common.utils.resources.ResourceUtils;
 import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicy;
 import org.onap.policy.models.tosca.authorative.concepts.ToscaServiceTemplate;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.yaml.snakeyaml.Yaml;
 
 public class TestUtils {
+    private static final Logger LOGGER = LoggerFactory.getLogger(TestUtils.class);
     private static final StandardCoder standardCoder = new StandardCoder();
 
     private TestUtils() {
@@ -47,8 +52,15 @@ public class TestUtils {
      * @throws CoderException exception if it cannot be decoded
      * @throws XacmlApplicationException If the application cannot load the policy
      */
-    public static void loadPolicies(String resourceFile, XacmlApplicationServiceProvider service)
+    public static List<ToscaPolicy> loadPolicies(String resourceFile, XacmlApplicationServiceProvider service)
             throws CoderException, XacmlApplicationException {
+        //
+        // Our return object
+        //
+        List<ToscaPolicy> loadedPolicies = new ArrayList<>();
+        //
+        // Decode it
+        //
         String policyYaml = ResourceUtils.getResourceAsString(resourceFile);
         Yaml yaml = new Yaml();
         Object yamlObject = yaml.load(policyYaml);
@@ -58,11 +70,15 @@ public class TestUtils {
         // Get the policies
         //
         for (Map<String, ToscaPolicy> policies : serviceTemplate.getToscaTopologyTemplate().getPolicies()) {
-            for (Entry<String, ToscaPolicy> entrySet : policies.entrySet()) {
-                service.loadPolicy(entrySet.getValue());
+            for (ToscaPolicy policy : policies.values()) {
+                if (service.loadPolicy(policy)) {
+                    loadedPolicies.add(policy);
+                } else {
+                    LOGGER.error("Application failed to load policy");
+                }
             }
         }
-
+        return loadedPolicies;
     }
 
 }
index c1682fb..d4cce5c 100644 (file)
@@ -83,7 +83,14 @@ public interface XacmlApplicationServiceProvider {
      *
      * @param toscaPolicy object
      */
-    void             loadPolicy(ToscaPolicy toscaPolicy) throws XacmlApplicationException;
+    boolean          loadPolicy(ToscaPolicy toscaPolicy) throws XacmlApplicationException;
+
+    /**
+     * unloadPolicy a Tosca Policy.
+     *
+     * @param toscaPolicy object
+     */
+    boolean          unloadPolicy(ToscaPolicy toscaPolicy) throws XacmlApplicationException;
 
     /**
      * Makes a decision given the incoming request and returns a response.
index 364b651..2a5f21f 100644 (file)
@@ -214,6 +214,50 @@ public class XacmlPolicyUtils {
         return properties;
     }
 
+    /**
+     * Removes a root policy from the Properties object. Both in the line
+     * that identifies the policy and the .file property that points to the path.
+     *
+     * @param properties Input Properties object to remove
+     * @param rootPolicyPath The policy file path
+     * @return Properties object
+     */
+    public static Properties removeRootPolicy(Properties properties, Path rootPolicyPath) {
+        //
+        // Get the current set of referenced policy ids
+        //
+        StringJoiner join = new StringJoiner(",");
+        boolean found = false;
+        Set<String> rootPolicies = XACMLProperties.getRootPolicyIDs(properties);
+        for (String refPolicy : rootPolicies) {
+            String refPolicyFile = refPolicy + DOT_FILE_SUFFIX;
+            //
+            // If the key and value match, then it will return true
+            //
+            if (properties.remove(refPolicyFile, rootPolicyPath.toString())) {
+                //
+                // Record that we actually removed it
+                //
+                found = true;
+            } else {
+                //
+                // Retain it
+                //
+                join.add(refPolicy);
+            }
+        }
+        //
+        // Did we remove it?
+        //
+        if (found) {
+            //
+            // Now update the list of referenced properties
+            //
+            properties.setProperty(XACMLProperties.PROP_ROOTPOLICIES, join.toString());
+        }
+        return properties;
+    }
+
     /**
      * Removes a referenced policy from the Properties object. Both in the line
      * that identifies the policy and the .file property that points to the path.
index 7f85d2f..2eddc2e 100644 (file)
@@ -37,7 +37,9 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 
 import oasis.names.tc.xacml._3_0.core.schema.wd_17.PolicyType;
@@ -60,6 +62,7 @@ public abstract class StdXacmlApplicationServiceProvider implements XacmlApplica
     private Path pathForData = null;
     private Properties pdpProperties = null;
     private PDPEngine pdpEngine = null;
+    private Map<ToscaPolicy, Path> mapLoadedPolicies = new HashMap<>();
 
     public StdXacmlApplicationServiceProvider() {
         super();
@@ -115,7 +118,7 @@ public abstract class StdXacmlApplicationServiceProvider implements XacmlApplica
 
     @Override
     public List<ToscaPolicyTypeIdentifier> supportedPolicyTypes() {
-        return Collections.emptyList();
+        throw new UnsupportedOperationException("Please override and implement supportedPolicyTypes");
     }
 
     @Override
@@ -124,7 +127,7 @@ public abstract class StdXacmlApplicationServiceProvider implements XacmlApplica
     }
 
     @Override
-    public synchronized void loadPolicy(ToscaPolicy toscaPolicy) {
+    public synchronized boolean loadPolicy(ToscaPolicy toscaPolicy) {
         try {
             //
             // Convert the policies first
@@ -166,13 +169,77 @@ public abstract class StdXacmlApplicationServiceProvider implements XacmlApplica
             // Save the properties
             //
             this.pdpProperties = newProperties;
+            //
+            // Save in our map
+            //
+            this.mapLoadedPolicies.put(toscaPolicy, refPath);
         } catch (IOException | ToscaPolicyConversionException e) {
             LOGGER.error("Failed to loadPolicies {}", e);
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public synchronized boolean unloadPolicy(ToscaPolicy toscaPolicy) throws XacmlApplicationException {
+        //
+        // Find it in our map
+        //
+        Path refPolicy = this.mapLoadedPolicies.get(toscaPolicy);
+        if (refPolicy == null) {
+            LOGGER.error("Failed to find ToscaPolicy {} in our map size {}", toscaPolicy.getMetadata(),
+                    this.mapLoadedPolicies.size());
+            return false;
+        }
+        //
+        // Create a copy of the properties object
+        //
+        Properties newProperties = this.getProperties();
+        //
+        // Remove it from the properties
+        //
+        XacmlPolicyUtils.removeRootPolicy(newProperties, refPolicy);
+        //
+        // We can delete the file
+        //
+        try {
+            Files.delete(refPolicy);
+        } catch (IOException e) {
+            LOGGER.error("Failed to delete policy {} from disk {}", toscaPolicy.getMetadata(),
+                    refPolicy.toAbsolutePath().toString(), e);
         }
+        //
+        // Write the properties to disk
+        //
+        try {
+            XacmlPolicyUtils.storeXacmlProperties(newProperties,
+                    XacmlPolicyUtils.getPropertiesPath(this.getDataPath()));
+        } catch (IOException e) {
+            LOGGER.error("Failed to save the properties to disk {}", newProperties);
+        }
+        //
+        // Reload the engine
+        //
+        this.createEngine(newProperties);
+        //
+        // Save the properties
+        //
+        this.pdpProperties = newProperties;
+        //
+        // Save in our map
+        //
+        if (this.mapLoadedPolicies.remove(toscaPolicy) == null) {
+            LOGGER.error("Failed to remove toscaPolicy {} from internal map size {}", toscaPolicy.getMetadata(),
+                    this.mapLoadedPolicies.size());
+        }
+        //
+        // Not sure if any of the errors above warrant returning false
+        //
+        return true;
     }
 
     @Override
-    public synchronized DecisionResponse makeDecision(DecisionRequest request) {
+    public DecisionResponse makeDecision(DecisionRequest request) {
         //
         // Convert to a XacmlRequest
         //
@@ -251,7 +318,6 @@ public abstract class StdXacmlApplicationServiceProvider implements XacmlApplica
             PDPEngine engine = factory.newEngine(properties);
             if (engine != null) {
                 this.pdpEngine = engine;
-//                this.pdpProperties = new Properties(properties);
             }
         } catch (FactoryException e) {
             LOGGER.error("Failed to create XACML PDP Engine {}", e);
index fe0f675..ed63bb9 100644 (file)
@@ -225,7 +225,7 @@ public class XacmlPolicyUtilsTest {
     }
 
     @Test
-    public void testRemovingProperties() {
+    public void testRemovingReferencedProperties() {
         //
         // Dump what we are starting with
         //
@@ -261,4 +261,27 @@ public class XacmlPolicyUtilsTest {
         XacmlPolicyUtils.debugDumpPolicyProperties(properties, LOGGER);
         assertThat(properties.getProperty("refstart4.file")).isNullOrEmpty();
     }
+
+    @Test
+    public void testRemovingRootProperties() {
+        //
+        // Dump what we are starting with
+        //
+        XacmlPolicyUtils.debugDumpPolicyProperties(properties, LOGGER);
+        //
+        // Remove root policies
+        //
+        Path ref = Paths.get("src/test/resources/root.xml");
+        XacmlPolicyUtils.removeRootPolicy(properties, ref);
+        XacmlPolicyUtils.debugDumpPolicyProperties(properties, LOGGER);
+        assertThat(properties.getProperty("root.file")).isNullOrEmpty();
+
+        //
+        // Test one that isn't in there
+        //
+        ref = Paths.get("src/test/resources/NotThere.xml");
+        XacmlPolicyUtils.removeRootPolicy(properties, ref);
+        XacmlPolicyUtils.debugDumpPolicyProperties(properties, LOGGER);
+        assertThat(properties.getProperty("refstart3.file")).isNotBlank();
+    }
 }
index cc11dcf..9b26df1 100644 (file)
@@ -27,6 +27,7 @@ import static org.assertj.core.api.Assertions.assertThat;
 import java.io.File;
 import java.io.IOException;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Properties;
 import java.util.ServiceLoader;
 
@@ -41,6 +42,7 @@ import org.onap.policy.common.utils.coder.StandardCoder;
 import org.onap.policy.common.utils.resources.TextFileUtils;
 import org.onap.policy.models.decisions.concepts.DecisionRequest;
 import org.onap.policy.models.decisions.concepts.DecisionResponse;
+import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicy;
 import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicyTypeIdentifier;
 import org.onap.policy.pdp.xacml.application.common.TestUtils;
 import org.onap.policy.pdp.xacml.application.common.XacmlApplicationException;
@@ -161,7 +163,8 @@ public class MonitoringPdpApplicationTest {
         //
         // Now load the optimization policies
         //
-        TestUtils.loadPolicies("src/test/resources/vDNS.policy.input.yaml", service);
+        final List<ToscaPolicy> loadedPolicies = TestUtils.loadPolicies("src/test/resources/vDNS.policy.input.yaml",
+                service);
         //
         // Ask for a decision
         //
@@ -174,6 +177,21 @@ public class MonitoringPdpApplicationTest {
         // Dump it out as Json
         //
         LOGGER.info(gson.encode(response));
+        LOGGER.info("Now testing unloading of policy");
+        //
+        // Now unload it
+        //
+        for (ToscaPolicy policy : loadedPolicies) {
+            assertThat(service.unloadPolicy(policy)).isTrue();
+        }
+        //
+        // Ask for a decision
+        //
+        response = service.makeDecision(requestSinglePolicy);
+        LOGGER.info("Decision {}", response);
+
+        assertThat(response).isNotNull();
+        assertThat(response.getPolicies().size()).isEqualTo(0);
     }
 
     @Test
index 809e43d..1253ff2 100644 (file)
@@ -22,7 +22,6 @@
 
 package org.onap.policy.pdpx.main.comm;
 
-import java.net.UnknownHostException;
 import org.onap.policy.common.utils.network.NetworkUtil;
 import org.onap.policy.models.pdp.concepts.PdpStateChange;
 import org.onap.policy.models.pdp.concepts.PdpStatus;
@@ -111,7 +110,7 @@ public class XacmlPdpMessage {
         status.setPdpGroup(message.getPdpGroup());
         status.setPdpSubgroup(message.getPdpSubgroup());
         status.setSupportedPolicyTypes(XacmlPdpApplicationManager.getToscaPolicyTypeIdents());
-        status.setPolicies(XacmlPdpApplicationManager.getToscaPolicies());
+        status.setPolicies(XacmlPdpApplicationManager.getToscaPolicyIdentifiers());
 
         return status;
     }
index 716421c..4c9d0c2 100644 (file)
@@ -20,6 +20,8 @@
 
 package org.onap.policy.pdpx.main.comm;
 
+import java.util.HashSet;
+import java.util.Set;
 import org.onap.policy.common.endpoints.event.comm.client.TopicSinkClient;
 import org.onap.policy.models.pdp.concepts.PdpStatus;
 import org.onap.policy.models.pdp.concepts.PdpUpdate;
@@ -45,9 +47,26 @@ public class XacmlPdpUpdatePublisher {
     public static void handlePdpUpdate(PdpUpdate message, TopicSinkClient client) {
 
         if (!message.getPolicies().isEmpty() || message.getPolicies() != null) {
-            // Load the policies on PDP applications
-            for (ToscaPolicy toscaPolicy : message.getPolicies()) {
-                XacmlPdpApplicationManager.loadDeployedPolicy(toscaPolicy);
+
+            Set<ToscaPolicy> incomingPolicies =
+                    new HashSet<>(message.getPolicies());
+            Set<ToscaPolicy> deployedPolicies =
+                    new HashSet<>(XacmlPdpApplicationManager.getToscaPolicies().keySet());
+
+            // Undeploy a policy
+            // if incoming policies do not contain the deployed policy then remove it from PDP
+            for (ToscaPolicy policy : deployedPolicies) {
+                if (!incomingPolicies.contains(policy)) {
+                    XacmlPdpApplicationManager.removeUndeployedPolicy(policy);
+                }
+            }
+
+            // Deploy a policy
+            // if deployed policies do not contain the incoming policy load it
+            for (ToscaPolicy policy : incomingPolicies) {
+                if (!deployedPolicies.contains(policy)) {
+                    XacmlPdpApplicationManager.loadDeployedPolicy(policy);
+                }
             }
         }
 
index 0980559..785d759 100644 (file)
@@ -29,6 +29,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.ServiceLoader;
+import java.util.stream.Collectors;
 import org.onap.policy.models.decisions.concepts.DecisionRequest;
 import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicy;
 import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicyIdentifier;
@@ -45,7 +46,10 @@ public class XacmlPdpApplicationManager {
     private static ServiceLoader<XacmlApplicationServiceProvider> applicationLoader;
     private static Map<String, XacmlApplicationServiceProvider> providerActionMap = new HashMap<>();
     private static List<ToscaPolicyTypeIdentifier> toscaPolicyTypeIdents = new ArrayList<>();
-    private static List<ToscaPolicyIdentifier> toscaPolicies = new ArrayList<>();
+    private static List<ToscaPolicyIdentifier> toscaPolicyIdents = new ArrayList<>();
+    private static List<ToscaPolicy> toscaPolicies = new ArrayList<>();
+    private static Map<ToscaPolicy, XacmlApplicationServiceProvider> mapLoadedPolicies = new HashMap<>();
+
 
     private XacmlPdpApplicationManager() {
         super();
@@ -115,10 +119,54 @@ public class XacmlPdpApplicationManager {
         return providerActionMap.get(request.getAction());
     }
 
+    /**
+     * getToscaPolicies.
+     *
+     * @return the map containing ToscaPolicies
+     */
+    public static Map<ToscaPolicy, XacmlApplicationServiceProvider> getToscaPolicies() {
+        return mapLoadedPolicies;
+    }
+
+    /**
+     * getToscaPolicyIdentifiers.
+     *
+     * @return list of ToscaPolicyIdentifier
+     */
+    public static List<ToscaPolicyIdentifier> getToscaPolicyIdentifiers() {
+        //
+        // converting map to return List of ToscaPolicyIdentiers
+        //
+        return mapLoadedPolicies.keySet().stream().map(ToscaPolicy::getIdentifier).collect(Collectors.toList());
+    }
+
     public static List<ToscaPolicyTypeIdentifier> getToscaPolicyTypeIdents() {
         return toscaPolicyTypeIdents;
     }
 
+    /**
+     * Finds the appropriate application and removes the policy.
+     *
+     * @param policy Incoming policy
+     */
+    public static void removeUndeployedPolicy(ToscaPolicy policy) {
+
+        for (XacmlApplicationServiceProvider application : applicationLoader) {
+            try {
+                if (application.unloadPolicy(policy)) {
+                    LOGGER.info("Unloaded ToscaPolicy {} from application {}", policy.getMetadata(),
+                            application.applicationName());
+                    if (mapLoadedPolicies.remove(policy) == null) {
+                        LOGGER.error("Failed to remove unloaded policy {} from map size {}", policy.getMetadata(),
+                                mapLoadedPolicies.size());
+                    }
+                }
+            } catch (XacmlApplicationException e) {
+                LOGGER.error("Failed to undeploy the Tosca Policy", e);
+            }
+        }
+    }
+
     /**
      * Finds the appropriate application and loads the policy.
      *
@@ -134,7 +182,11 @@ public class XacmlPdpApplicationManager {
                 // just use the first one found.
                 //
                 if (application.canSupportPolicyType(policy.getTypeIdentifier())) {
-                    application.loadPolicy(policy);
+                    if (application.loadPolicy(policy)) {
+                        LOGGER.info("Loaded ToscaPolicy {} into application {}", policy.getMetadata(),
+                                application.applicationName());
+                        mapLoadedPolicies.put(policy, application);
+                    }
                     return;
                 }
             } catch (XacmlApplicationException e) {
@@ -143,10 +195,6 @@ public class XacmlPdpApplicationManager {
         }
     }
 
-    public static List<ToscaPolicyIdentifier> getToscaPolicies() {
-        return toscaPolicies;
-    }
-
     /**
      * Returns the current count of policy types supported. This could be misleading a bit
      * as some applications can support wildcard of policy types. Eg. onap.Monitoring.* as
index e08be24..768709f 100644 (file)
             <outputDirectory>/etc/ssl</outputDirectory>
             <lineEnding>keep</lineEnding>
         </fileSet>
+        <fileSet>
+            <directory>${project.basedir}/src/main/resources/apps/guard
+            </directory>
+            <includes>
+                <include>*.properties</include>
+            </includes>
+            <outputDirectory>/apps/guard</outputDirectory>
+            <lineEnding>unix</lineEnding>
+        </fileSet>
+                <fileSet>
+            <directory>${project.basedir}/src/main/resources/apps/monitoring
+            </directory>
+            <includes>
+                <include>*.properties</include>
+            </includes>
+            <outputDirectory>/apps/monitoring</outputDirectory>
+            <lineEnding>unix</lineEnding>
+        </fileSet>
+                <fileSet>
+            <directory>${project.basedir}/src/main/resources/apps/optimization
+            </directory>
+            <includes>
+                <include>*.properties</include>
+            </includes>
+            <outputDirectory>/apps/optimization</outputDirectory>
+            <lineEnding>unix</lineEnding>
+        </fileSet>
     </fileSets>
 </assembly>