From 432cec9fa6f143dad324cd11f62fb052c7da32b7 Mon Sep 17 00:00:00 2001 From: Michael Mokry Date: Thu, 11 Apr 2019 12:19:50 -0500 Subject: [PATCH] Changes to handle PDPX deploy/undeploy Added changes to handle incoming udpate for deploying or undeploying the policies by comparing both sets. STill need to add support for removing the policies and getting the deployed ToscaPolicyIdentifiers for PdpStatus response. * PLD - added functionality to unload policies in the applications and * PLD - added map of loaded policies * Mike - made change to address Jim's and Joshua's comments * Mike - Made change to use mapLoadedPolicies to get deployed policies from the applications * Mike - made change to assembly.xml to add support for apps directory. Change-Id: If7d119197171b68b905b7fd0f2b5be6cf6f609e7 Issue-ID: POLICY-1451 Signed-off-by: Michael Mokry Signed-off-by: Pamela Dragosh --- .../pdp/xacml/application/common/TestUtils.java | 24 +++++-- .../common/XacmlApplicationServiceProvider.java | 9 ++- .../xacml/application/common/XacmlPolicyUtils.java | 44 +++++++++++++ .../std/StdXacmlApplicationServiceProvider.java | 74 ++++++++++++++++++++-- .../application/common/XacmlPolicyUtilsTest.java | 25 +++++++- .../monitoring/MonitoringPdpApplicationTest.java | 20 +++++- .../policy/pdpx/main/comm/XacmlPdpMessage.java | 3 +- .../pdpx/main/comm/XacmlPdpUpdatePublisher.java | 25 +++++++- .../pdpx/main/rest/XacmlPdpApplicationManager.java | 60 ++++++++++++++++-- .../src/main/package/tarball/assembly.xml | 27 ++++++++ 10 files changed, 289 insertions(+), 22 deletions(-) diff --git a/applications/common/src/main/java/org/onap/policy/pdp/xacml/application/common/TestUtils.java b/applications/common/src/main/java/org/onap/policy/pdp/xacml/application/common/TestUtils.java index fa32516d..50eb50bf 100644 --- a/applications/common/src/main/java/org/onap/policy/pdp/xacml/application/common/TestUtils.java +++ b/applications/common/src/main/java/org/onap/policy/pdp/xacml/application/common/TestUtils.java @@ -22,6 +22,8 @@ package org.onap.policy.pdp.xacml.application.common; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -30,9 +32,12 @@ import org.onap.policy.common.utils.coder.StandardCoder; import org.onap.policy.common.utils.resources.ResourceUtils; import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicy; import org.onap.policy.models.tosca.authorative.concepts.ToscaServiceTemplate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.Yaml; public class TestUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(TestUtils.class); private static final StandardCoder standardCoder = new StandardCoder(); private TestUtils() { @@ -47,8 +52,15 @@ public class TestUtils { * @throws CoderException exception if it cannot be decoded * @throws XacmlApplicationException If the application cannot load the policy */ - public static void loadPolicies(String resourceFile, XacmlApplicationServiceProvider service) + public static List loadPolicies(String resourceFile, XacmlApplicationServiceProvider service) throws CoderException, XacmlApplicationException { + // + // Our return object + // + List loadedPolicies = new ArrayList<>(); + // + // Decode it + // String policyYaml = ResourceUtils.getResourceAsString(resourceFile); Yaml yaml = new Yaml(); Object yamlObject = yaml.load(policyYaml); @@ -58,11 +70,15 @@ public class TestUtils { // Get the policies // for (Map policies : serviceTemplate.getToscaTopologyTemplate().getPolicies()) { - for (Entry entrySet : policies.entrySet()) { - service.loadPolicy(entrySet.getValue()); + for (ToscaPolicy policy : policies.values()) { + if (service.loadPolicy(policy)) { + loadedPolicies.add(policy); + } else { + LOGGER.error("Application failed to load policy"); + } } } - + return loadedPolicies; } } diff --git a/applications/common/src/main/java/org/onap/policy/pdp/xacml/application/common/XacmlApplicationServiceProvider.java b/applications/common/src/main/java/org/onap/policy/pdp/xacml/application/common/XacmlApplicationServiceProvider.java index c1682fb7..d4cce5c1 100644 --- a/applications/common/src/main/java/org/onap/policy/pdp/xacml/application/common/XacmlApplicationServiceProvider.java +++ b/applications/common/src/main/java/org/onap/policy/pdp/xacml/application/common/XacmlApplicationServiceProvider.java @@ -83,7 +83,14 @@ public interface XacmlApplicationServiceProvider { * * @param toscaPolicy object */ - void loadPolicy(ToscaPolicy toscaPolicy) throws XacmlApplicationException; + boolean loadPolicy(ToscaPolicy toscaPolicy) throws XacmlApplicationException; + + /** + * unloadPolicy a Tosca Policy. + * + * @param toscaPolicy object + */ + boolean unloadPolicy(ToscaPolicy toscaPolicy) throws XacmlApplicationException; /** * Makes a decision given the incoming request and returns a response. diff --git a/applications/common/src/main/java/org/onap/policy/pdp/xacml/application/common/XacmlPolicyUtils.java b/applications/common/src/main/java/org/onap/policy/pdp/xacml/application/common/XacmlPolicyUtils.java index 364b6519..2a5f21f0 100644 --- a/applications/common/src/main/java/org/onap/policy/pdp/xacml/application/common/XacmlPolicyUtils.java +++ b/applications/common/src/main/java/org/onap/policy/pdp/xacml/application/common/XacmlPolicyUtils.java @@ -214,6 +214,50 @@ public class XacmlPolicyUtils { return properties; } + /** + * Removes a root policy from the Properties object. Both in the line + * that identifies the policy and the .file property that points to the path. + * + * @param properties Input Properties object to remove + * @param rootPolicyPath The policy file path + * @return Properties object + */ + public static Properties removeRootPolicy(Properties properties, Path rootPolicyPath) { + // + // Get the current set of referenced policy ids + // + StringJoiner join = new StringJoiner(","); + boolean found = false; + Set rootPolicies = XACMLProperties.getRootPolicyIDs(properties); + for (String refPolicy : rootPolicies) { + String refPolicyFile = refPolicy + DOT_FILE_SUFFIX; + // + // If the key and value match, then it will return true + // + if (properties.remove(refPolicyFile, rootPolicyPath.toString())) { + // + // Record that we actually removed it + // + found = true; + } else { + // + // Retain it + // + join.add(refPolicy); + } + } + // + // Did we remove it? + // + if (found) { + // + // Now update the list of referenced properties + // + properties.setProperty(XACMLProperties.PROP_ROOTPOLICIES, join.toString()); + } + return properties; + } + /** * Removes a referenced policy from the Properties object. Both in the line * that identifies the policy and the .file property that points to the path. diff --git a/applications/common/src/main/java/org/onap/policy/pdp/xacml/application/common/std/StdXacmlApplicationServiceProvider.java b/applications/common/src/main/java/org/onap/policy/pdp/xacml/application/common/std/StdXacmlApplicationServiceProvider.java index 7f85d2f0..2eddc2e8 100644 --- a/applications/common/src/main/java/org/onap/policy/pdp/xacml/application/common/std/StdXacmlApplicationServiceProvider.java +++ b/applications/common/src/main/java/org/onap/policy/pdp/xacml/application/common/std/StdXacmlApplicationServiceProvider.java @@ -37,7 +37,9 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Properties; import oasis.names.tc.xacml._3_0.core.schema.wd_17.PolicyType; @@ -60,6 +62,7 @@ public abstract class StdXacmlApplicationServiceProvider implements XacmlApplica private Path pathForData = null; private Properties pdpProperties = null; private PDPEngine pdpEngine = null; + private Map mapLoadedPolicies = new HashMap<>(); public StdXacmlApplicationServiceProvider() { super(); @@ -115,7 +118,7 @@ public abstract class StdXacmlApplicationServiceProvider implements XacmlApplica @Override public List supportedPolicyTypes() { - return Collections.emptyList(); + throw new UnsupportedOperationException("Please override and implement supportedPolicyTypes"); } @Override @@ -124,7 +127,7 @@ public abstract class StdXacmlApplicationServiceProvider implements XacmlApplica } @Override - public synchronized void loadPolicy(ToscaPolicy toscaPolicy) { + public synchronized boolean loadPolicy(ToscaPolicy toscaPolicy) { try { // // Convert the policies first @@ -166,13 +169,77 @@ public abstract class StdXacmlApplicationServiceProvider implements XacmlApplica // Save the properties // this.pdpProperties = newProperties; + // + // Save in our map + // + this.mapLoadedPolicies.put(toscaPolicy, refPath); } catch (IOException | ToscaPolicyConversionException e) { LOGGER.error("Failed to loadPolicies {}", e); + return false; + } + return true; + } + + @Override + public synchronized boolean unloadPolicy(ToscaPolicy toscaPolicy) throws XacmlApplicationException { + // + // Find it in our map + // + Path refPolicy = this.mapLoadedPolicies.get(toscaPolicy); + if (refPolicy == null) { + LOGGER.error("Failed to find ToscaPolicy {} in our map size {}", toscaPolicy.getMetadata(), + this.mapLoadedPolicies.size()); + return false; + } + // + // Create a copy of the properties object + // + Properties newProperties = this.getProperties(); + // + // Remove it from the properties + // + XacmlPolicyUtils.removeRootPolicy(newProperties, refPolicy); + // + // We can delete the file + // + try { + Files.delete(refPolicy); + } catch (IOException e) { + LOGGER.error("Failed to delete policy {} from disk {}", toscaPolicy.getMetadata(), + refPolicy.toAbsolutePath().toString(), e); } + // + // Write the properties to disk + // + try { + XacmlPolicyUtils.storeXacmlProperties(newProperties, + XacmlPolicyUtils.getPropertiesPath(this.getDataPath())); + } catch (IOException e) { + LOGGER.error("Failed to save the properties to disk {}", newProperties); + } + // + // Reload the engine + // + this.createEngine(newProperties); + // + // Save the properties + // + this.pdpProperties = newProperties; + // + // Save in our map + // + if (this.mapLoadedPolicies.remove(toscaPolicy) == null) { + LOGGER.error("Failed to remove toscaPolicy {} from internal map size {}", toscaPolicy.getMetadata(), + this.mapLoadedPolicies.size()); + } + // + // Not sure if any of the errors above warrant returning false + // + return true; } @Override - public synchronized DecisionResponse makeDecision(DecisionRequest request) { + public DecisionResponse makeDecision(DecisionRequest request) { // // Convert to a XacmlRequest // @@ -251,7 +318,6 @@ public abstract class StdXacmlApplicationServiceProvider implements XacmlApplica PDPEngine engine = factory.newEngine(properties); if (engine != null) { this.pdpEngine = engine; -// this.pdpProperties = new Properties(properties); } } catch (FactoryException e) { LOGGER.error("Failed to create XACML PDP Engine {}", e); diff --git a/applications/common/src/test/java/org/onap/policy/pdp/xacml/application/common/XacmlPolicyUtilsTest.java b/applications/common/src/test/java/org/onap/policy/pdp/xacml/application/common/XacmlPolicyUtilsTest.java index fe0f675d..ed63bb9c 100644 --- a/applications/common/src/test/java/org/onap/policy/pdp/xacml/application/common/XacmlPolicyUtilsTest.java +++ b/applications/common/src/test/java/org/onap/policy/pdp/xacml/application/common/XacmlPolicyUtilsTest.java @@ -225,7 +225,7 @@ public class XacmlPolicyUtilsTest { } @Test - public void testRemovingProperties() { + public void testRemovingReferencedProperties() { // // Dump what we are starting with // @@ -261,4 +261,27 @@ public class XacmlPolicyUtilsTest { XacmlPolicyUtils.debugDumpPolicyProperties(properties, LOGGER); assertThat(properties.getProperty("refstart4.file")).isNullOrEmpty(); } + + @Test + public void testRemovingRootProperties() { + // + // Dump what we are starting with + // + XacmlPolicyUtils.debugDumpPolicyProperties(properties, LOGGER); + // + // Remove root policies + // + Path ref = Paths.get("src/test/resources/root.xml"); + XacmlPolicyUtils.removeRootPolicy(properties, ref); + XacmlPolicyUtils.debugDumpPolicyProperties(properties, LOGGER); + assertThat(properties.getProperty("root.file")).isNullOrEmpty(); + + // + // Test one that isn't in there + // + ref = Paths.get("src/test/resources/NotThere.xml"); + XacmlPolicyUtils.removeRootPolicy(properties, ref); + XacmlPolicyUtils.debugDumpPolicyProperties(properties, LOGGER); + assertThat(properties.getProperty("refstart3.file")).isNotBlank(); + } } diff --git a/applications/monitoring/src/test/java/org/onap/policy/xacml/pdp/application/monitoring/MonitoringPdpApplicationTest.java b/applications/monitoring/src/test/java/org/onap/policy/xacml/pdp/application/monitoring/MonitoringPdpApplicationTest.java index cc11dcf6..9b26df16 100644 --- a/applications/monitoring/src/test/java/org/onap/policy/xacml/pdp/application/monitoring/MonitoringPdpApplicationTest.java +++ b/applications/monitoring/src/test/java/org/onap/policy/xacml/pdp/application/monitoring/MonitoringPdpApplicationTest.java @@ -27,6 +27,7 @@ import static org.assertj.core.api.Assertions.assertThat; import java.io.File; import java.io.IOException; import java.util.Iterator; +import java.util.List; import java.util.Properties; import java.util.ServiceLoader; @@ -41,6 +42,7 @@ import org.onap.policy.common.utils.coder.StandardCoder; import org.onap.policy.common.utils.resources.TextFileUtils; import org.onap.policy.models.decisions.concepts.DecisionRequest; import org.onap.policy.models.decisions.concepts.DecisionResponse; +import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicy; import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicyTypeIdentifier; import org.onap.policy.pdp.xacml.application.common.TestUtils; import org.onap.policy.pdp.xacml.application.common.XacmlApplicationException; @@ -161,7 +163,8 @@ public class MonitoringPdpApplicationTest { // // Now load the optimization policies // - TestUtils.loadPolicies("src/test/resources/vDNS.policy.input.yaml", service); + final List loadedPolicies = TestUtils.loadPolicies("src/test/resources/vDNS.policy.input.yaml", + service); // // Ask for a decision // @@ -174,6 +177,21 @@ public class MonitoringPdpApplicationTest { // Dump it out as Json // LOGGER.info(gson.encode(response)); + LOGGER.info("Now testing unloading of policy"); + // + // Now unload it + // + for (ToscaPolicy policy : loadedPolicies) { + assertThat(service.unloadPolicy(policy)).isTrue(); + } + // + // Ask for a decision + // + response = service.makeDecision(requestSinglePolicy); + LOGGER.info("Decision {}", response); + + assertThat(response).isNotNull(); + assertThat(response.getPolicies().size()).isEqualTo(0); } @Test diff --git a/main/src/main/java/org/onap/policy/pdpx/main/comm/XacmlPdpMessage.java b/main/src/main/java/org/onap/policy/pdpx/main/comm/XacmlPdpMessage.java index 809e43d0..1253ff28 100644 --- a/main/src/main/java/org/onap/policy/pdpx/main/comm/XacmlPdpMessage.java +++ b/main/src/main/java/org/onap/policy/pdpx/main/comm/XacmlPdpMessage.java @@ -22,7 +22,6 @@ package org.onap.policy.pdpx.main.comm; -import java.net.UnknownHostException; import org.onap.policy.common.utils.network.NetworkUtil; import org.onap.policy.models.pdp.concepts.PdpStateChange; import org.onap.policy.models.pdp.concepts.PdpStatus; @@ -111,7 +110,7 @@ public class XacmlPdpMessage { status.setPdpGroup(message.getPdpGroup()); status.setPdpSubgroup(message.getPdpSubgroup()); status.setSupportedPolicyTypes(XacmlPdpApplicationManager.getToscaPolicyTypeIdents()); - status.setPolicies(XacmlPdpApplicationManager.getToscaPolicies()); + status.setPolicies(XacmlPdpApplicationManager.getToscaPolicyIdentifiers()); return status; } diff --git a/main/src/main/java/org/onap/policy/pdpx/main/comm/XacmlPdpUpdatePublisher.java b/main/src/main/java/org/onap/policy/pdpx/main/comm/XacmlPdpUpdatePublisher.java index 716421c7..4c9d0c21 100644 --- a/main/src/main/java/org/onap/policy/pdpx/main/comm/XacmlPdpUpdatePublisher.java +++ b/main/src/main/java/org/onap/policy/pdpx/main/comm/XacmlPdpUpdatePublisher.java @@ -20,6 +20,8 @@ package org.onap.policy.pdpx.main.comm; +import java.util.HashSet; +import java.util.Set; import org.onap.policy.common.endpoints.event.comm.client.TopicSinkClient; import org.onap.policy.models.pdp.concepts.PdpStatus; import org.onap.policy.models.pdp.concepts.PdpUpdate; @@ -45,9 +47,26 @@ public class XacmlPdpUpdatePublisher { public static void handlePdpUpdate(PdpUpdate message, TopicSinkClient client) { if (!message.getPolicies().isEmpty() || message.getPolicies() != null) { - // Load the policies on PDP applications - for (ToscaPolicy toscaPolicy : message.getPolicies()) { - XacmlPdpApplicationManager.loadDeployedPolicy(toscaPolicy); + + Set incomingPolicies = + new HashSet<>(message.getPolicies()); + Set deployedPolicies = + new HashSet<>(XacmlPdpApplicationManager.getToscaPolicies().keySet()); + + // Undeploy a policy + // if incoming policies do not contain the deployed policy then remove it from PDP + for (ToscaPolicy policy : deployedPolicies) { + if (!incomingPolicies.contains(policy)) { + XacmlPdpApplicationManager.removeUndeployedPolicy(policy); + } + } + + // Deploy a policy + // if deployed policies do not contain the incoming policy load it + for (ToscaPolicy policy : incomingPolicies) { + if (!deployedPolicies.contains(policy)) { + XacmlPdpApplicationManager.loadDeployedPolicy(policy); + } } } diff --git a/main/src/main/java/org/onap/policy/pdpx/main/rest/XacmlPdpApplicationManager.java b/main/src/main/java/org/onap/policy/pdpx/main/rest/XacmlPdpApplicationManager.java index 09805593..785d7591 100644 --- a/main/src/main/java/org/onap/policy/pdpx/main/rest/XacmlPdpApplicationManager.java +++ b/main/src/main/java/org/onap/policy/pdpx/main/rest/XacmlPdpApplicationManager.java @@ -29,6 +29,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.ServiceLoader; +import java.util.stream.Collectors; import org.onap.policy.models.decisions.concepts.DecisionRequest; import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicy; import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicyIdentifier; @@ -45,7 +46,10 @@ public class XacmlPdpApplicationManager { private static ServiceLoader applicationLoader; private static Map providerActionMap = new HashMap<>(); private static List toscaPolicyTypeIdents = new ArrayList<>(); - private static List toscaPolicies = new ArrayList<>(); + private static List toscaPolicyIdents = new ArrayList<>(); + private static List toscaPolicies = new ArrayList<>(); + private static Map mapLoadedPolicies = new HashMap<>(); + private XacmlPdpApplicationManager() { super(); @@ -115,10 +119,54 @@ public class XacmlPdpApplicationManager { return providerActionMap.get(request.getAction()); } + /** + * getToscaPolicies. + * + * @return the map containing ToscaPolicies + */ + public static Map getToscaPolicies() { + return mapLoadedPolicies; + } + + /** + * getToscaPolicyIdentifiers. + * + * @return list of ToscaPolicyIdentifier + */ + public static List getToscaPolicyIdentifiers() { + // + // converting map to return List of ToscaPolicyIdentiers + // + return mapLoadedPolicies.keySet().stream().map(ToscaPolicy::getIdentifier).collect(Collectors.toList()); + } + public static List getToscaPolicyTypeIdents() { return toscaPolicyTypeIdents; } + /** + * Finds the appropriate application and removes the policy. + * + * @param policy Incoming policy + */ + public static void removeUndeployedPolicy(ToscaPolicy policy) { + + for (XacmlApplicationServiceProvider application : applicationLoader) { + try { + if (application.unloadPolicy(policy)) { + LOGGER.info("Unloaded ToscaPolicy {} from application {}", policy.getMetadata(), + application.applicationName()); + if (mapLoadedPolicies.remove(policy) == null) { + LOGGER.error("Failed to remove unloaded policy {} from map size {}", policy.getMetadata(), + mapLoadedPolicies.size()); + } + } + } catch (XacmlApplicationException e) { + LOGGER.error("Failed to undeploy the Tosca Policy", e); + } + } + } + /** * Finds the appropriate application and loads the policy. * @@ -134,7 +182,11 @@ public class XacmlPdpApplicationManager { // just use the first one found. // if (application.canSupportPolicyType(policy.getTypeIdentifier())) { - application.loadPolicy(policy); + if (application.loadPolicy(policy)) { + LOGGER.info("Loaded ToscaPolicy {} into application {}", policy.getMetadata(), + application.applicationName()); + mapLoadedPolicies.put(policy, application); + } return; } } catch (XacmlApplicationException e) { @@ -143,10 +195,6 @@ public class XacmlPdpApplicationManager { } } - public static List getToscaPolicies() { - return toscaPolicies; - } - /** * Returns the current count of policy types supported. This could be misleading a bit * as some applications can support wildcard of policy types. Eg. onap.Monitoring.* as diff --git a/packages/policy-xacmlpdp-tarball/src/main/package/tarball/assembly.xml b/packages/policy-xacmlpdp-tarball/src/main/package/tarball/assembly.xml index e08be24d..768709f7 100644 --- a/packages/policy-xacmlpdp-tarball/src/main/package/tarball/assembly.xml +++ b/packages/policy-xacmlpdp-tarball/src/main/package/tarball/assembly.xml @@ -58,5 +58,32 @@ /etc/ssl keep + + ${project.basedir}/src/main/resources/apps/guard + + + *.properties + + /apps/guard + unix + + + ${project.basedir}/src/main/resources/apps/monitoring + + + *.properties + + /apps/monitoring + unix + + + ${project.basedir}/src/main/resources/apps/optimization + + + *.properties + + /apps/optimization + unix + -- 2.16.6