From 589e42747b6ad53680a8e51d64597c6590fa0ade Mon Sep 17 00:00:00 2001 From: liamfallon Date: Wed, 19 Feb 2020 16:54:34 +0000 Subject: [PATCH] Fix cascaded get filtering and speed This change adds proper cascading bahaviour to the returned service template for filtered gets on policies. It also significantly improves the performance on get operations, all get operations on the API now result in only one database query. Issue-ID: POLICY-1402 Change-Id: I7c7d0a8d9f1caf8b610461779e242c34b7f77365 Signed-off-by: liamfallon --- .../provider/impl/PolicyToscaPersistenceTest.java | 50 ++++++++++- .../provider/AuthorativeToscaProvider.java | 94 +++++++++++---------- .../tosca/simple/provider/SimpleToscaProvider.java | 98 +++++++++++++++++----- 3 files changed, 172 insertions(+), 70 deletions(-) diff --git a/models-provider/src/test/java/org/onap/policy/models/provider/impl/PolicyToscaPersistenceTest.java b/models-provider/src/test/java/org/onap/policy/models/provider/impl/PolicyToscaPersistenceTest.java index 311686831..11ecadb53 100644 --- a/models-provider/src/test/java/org/onap/policy/models/provider/impl/PolicyToscaPersistenceTest.java +++ b/models-provider/src/test/java/org/onap/policy/models/provider/impl/PolicyToscaPersistenceTest.java @@ -44,6 +44,7 @@ import org.onap.policy.models.provider.PolicyModelsProviderFactory; import org.onap.policy.models.provider.PolicyModelsProviderParameters; import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicy; import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicyFilter; +import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicyTypeFilter; import org.onap.policy.models.tosca.authorative.concepts.ToscaServiceTemplate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -107,6 +108,34 @@ public class PolicyToscaPersistenceTest { } } + @Test + public void testHpaPolicyTypeGet() throws PfModelException { + long getStartTime = System.currentTimeMillis(); + ToscaServiceTemplate hpaServiceTemplate = + databaseProvider.getPolicyTypes("onap.policies.optimization.resource.HpaPolicy", "1.0.0"); + LOGGER.trace("HPA policy normal get time (ms): {}", System.currentTimeMillis() - getStartTime); + + assertEquals(3, hpaServiceTemplate.getPolicyTypesAsMap().size()); + assertEquals(5, hpaServiceTemplate.getDataTypesAsMap().size()); + + getStartTime = System.currentTimeMillis(); + ToscaPolicyTypeFilter hpaFilter = ToscaPolicyTypeFilter.builder() + .name("onap.policies.optimization.resource.HpaPolicy").version("1.0.0").build(); + hpaServiceTemplate = databaseProvider.getFilteredPolicyTypes(hpaFilter); + LOGGER.trace("HPA policy filter name version get time (ms): {}", System.currentTimeMillis() - getStartTime); + + assertEquals(3, hpaServiceTemplate.getPolicyTypesAsMap().size()); + assertEquals(5, hpaServiceTemplate.getDataTypesAsMap().size()); + + getStartTime = System.currentTimeMillis(); + hpaFilter = ToscaPolicyTypeFilter.builder().name("onap.policies.optimization.resource.HpaPolicy").build(); + hpaServiceTemplate = databaseProvider.getFilteredPolicyTypes(hpaFilter); + LOGGER.trace("HPA policy filter name only get time (ms): {}", System.currentTimeMillis() - getStartTime); + + assertEquals(3, hpaServiceTemplate.getPolicyTypesAsMap().size()); + assertEquals(5, hpaServiceTemplate.getDataTypesAsMap().size()); + } + @Test public void testNamingPolicyGet() throws PfModelException { String policyYamlString = ResourceUtils.getResourceAsString("policies/sdnc.policy.naming.input.tosca.yaml"); @@ -120,7 +149,26 @@ public class PolicyToscaPersistenceTest { long getStartTime = System.currentTimeMillis(); ToscaServiceTemplate namingServiceTemplate = databaseProvider.getPolicies("SDNC_Policy.ONAP_VNF_NAMING_TIMESTAMP", "1.0.0"); - LOGGER.trace("Naming policy get time (ms): {}", System.currentTimeMillis() - getStartTime); + LOGGER.trace("Naming policy normal get time (ms): {}", System.currentTimeMillis() - getStartTime); + + assertEquals(1, namingServiceTemplate.getToscaTopologyTemplate().getPoliciesAsMap().size()); + assertEquals(1, namingServiceTemplate.getPolicyTypesAsMap().size()); + assertEquals(3, namingServiceTemplate.getDataTypesAsMap().size()); + + getStartTime = System.currentTimeMillis(); + ToscaPolicyFilter filter = + ToscaPolicyFilter.builder().name("SDNC_Policy.ONAP_VNF_NAMING_TIMESTAMP").version("1.0.0").build(); + namingServiceTemplate = databaseProvider.getFilteredPolicies(filter); + LOGGER.trace("Naming policy filtered get time (ms): {}", System.currentTimeMillis() - getStartTime); + + assertEquals(1, namingServiceTemplate.getToscaTopologyTemplate().getPoliciesAsMap().size()); + assertEquals(1, namingServiceTemplate.getPolicyTypesAsMap().size()); + assertEquals(3, namingServiceTemplate.getDataTypesAsMap().size()); + + getStartTime = System.currentTimeMillis(); + filter = ToscaPolicyFilter.builder().name("SDNC_Policy.ONAP_VNF_NAMING_TIMESTAMP").build(); + namingServiceTemplate = databaseProvider.getFilteredPolicies(filter); + LOGGER.trace("Naming policy filtered name only get time (ms): {}", System.currentTimeMillis() - getStartTime); assertEquals(1, namingServiceTemplate.getToscaTopologyTemplate().getPoliciesAsMap().size()); assertEquals(1, namingServiceTemplate.getPolicyTypesAsMap().size()); diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/provider/AuthorativeToscaProvider.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/provider/AuthorativeToscaProvider.java index a65cdbe69..65cce48db 100644 --- a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/provider/AuthorativeToscaProvider.java +++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/provider/AuthorativeToscaProvider.java @@ -23,15 +23,15 @@ package org.onap.policy.models.tosca.authorative.provider; import java.util.ArrayList; import java.util.Collections; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.TreeMap; +import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; import lombok.NonNull; +import org.apache.commons.collections4.CollectionUtils; import org.onap.policy.models.base.PfConceptKey; import org.onap.policy.models.base.PfModelException; import org.onap.policy.models.base.PfModelRuntimeException; @@ -44,6 +44,7 @@ import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicyTypeFilter; import org.onap.policy.models.tosca.authorative.concepts.ToscaServiceTemplate; import org.onap.policy.models.tosca.simple.concepts.JpaToscaServiceTemplate; import org.onap.policy.models.tosca.simple.provider.SimpleToscaProvider; +import org.onap.policy.models.tosca.utils.ToscaServiceTemplateUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -116,17 +117,33 @@ public class AuthorativeToscaProvider { @NonNull final ToscaPolicyTypeFilter filter) throws PfModelException { LOGGER.debug("->getFilteredPolicyTypes: filter={}", filter); + SimpleToscaProvider simpleToscaProvider = new SimpleToscaProvider(); - ToscaServiceTemplate serviceTemplate = - new SimpleToscaProvider().getPolicyTypes(dao, null, null).toAuthorative(); + final JpaToscaServiceTemplate dbServiceTemplate = simpleToscaProvider.getPolicyTypes(dao, null, null); - List filteredPolicyTypes = new ArrayList<>(serviceTemplate.getPolicyTypes().values()); + List filteredPolicyTypes = + new ArrayList<>(dbServiceTemplate.toAuthorative().getPolicyTypes().values()); filteredPolicyTypes = filter.filter(filteredPolicyTypes); - serviceTemplate.setPolicyTypes(asConceptMap(filteredPolicyTypes)); + if (CollectionUtils.isEmpty(filteredPolicyTypes)) { + throw new PfModelRuntimeException(Response.Status.NOT_FOUND, + "policy types for filter " + filter.toString() + " do not exist"); + } - LOGGER.debug("<-getFilteredPolicyTypes: filter={}, serviceTemplate={}", filter, serviceTemplate); - return serviceTemplate; + JpaToscaServiceTemplate filteredServiceTemplate = new JpaToscaServiceTemplate(); + + for (ToscaPolicyType policyType : filteredPolicyTypes) { + JpaToscaServiceTemplate cascadedServiceTemplate = simpleToscaProvider + .getCascadedPolicyTypes(dbServiceTemplate, policyType.getName(), policyType.getVersion()); + + filteredServiceTemplate = + ToscaServiceTemplateUtils.addFragment(filteredServiceTemplate, cascadedServiceTemplate); + } + + ToscaServiceTemplate returnServiceTemplate = filteredServiceTemplate.toAuthorative(); + + LOGGER.debug("<-getFilteredPolicyTypes: filter={}, serviceTemplate={}", filter, returnServiceTemplate); + return returnServiceTemplate; } /** @@ -272,16 +289,33 @@ public class AuthorativeToscaProvider { LOGGER.debug("->getFilteredPolicies: filter={}", filter); String version = ToscaPolicyFilter.LATEST_VERSION.equals(filter.getVersion()) ? null : filter.getVersion(); - ToscaServiceTemplate serviceTemplate = - new SimpleToscaProvider().getPolicies(dao, filter.getName(), version).toAuthorative(); + SimpleToscaProvider simpleToscaProvider = new SimpleToscaProvider(); + final JpaToscaServiceTemplate dbServiceTemplate = + simpleToscaProvider.getPolicies(dao, filter.getName(), version); - List filteredPolicies = asConceptList(serviceTemplate.getToscaTopologyTemplate().getPolicies()); + List filteredPolicies = + asConceptList(dbServiceTemplate.toAuthorative().getToscaTopologyTemplate().getPolicies()); filteredPolicies = filter.filter(filteredPolicies); - serviceTemplate.getToscaTopologyTemplate().setPolicies(asConceptMapList(filteredPolicies)); + if (CollectionUtils.isEmpty(filteredPolicies)) { + throw new PfModelRuntimeException(Response.Status.NOT_FOUND, + "policies for filter " + filter.toString() + " do not exist"); + } - LOGGER.debug("<-getFilteredPolicies: filter={}, serviceTemplate={}", filter, serviceTemplate); - return serviceTemplate; + JpaToscaServiceTemplate filteredServiceTemplate = new JpaToscaServiceTemplate(); + + for (ToscaPolicy policy : filteredPolicies) { + JpaToscaServiceTemplate cascadedServiceTemplate = + simpleToscaProvider.getCascadedPolicies(dbServiceTemplate, policy.getName(), policy.getVersion()); + + filteredServiceTemplate = + ToscaServiceTemplateUtils.addFragment(filteredServiceTemplate, cascadedServiceTemplate); + } + + ToscaServiceTemplate returnServiceTemplate = filteredServiceTemplate.toAuthorative(); + + LOGGER.debug("<-getFilteredPolicies: filter={}, serviceTemplate={}", filter, returnServiceTemplate); + return returnServiceTemplate; } /** @@ -383,38 +417,6 @@ public class AuthorativeToscaProvider { return returnList; } - /** - * Return the contents of a list of concepts as a list of maps of concepts. - * - * @param conceptList the concept list - * @return the list of concept map - */ - private List> asConceptMapList(List conceptList) { - List> toscaEntityMapList = new ArrayList<>(); - for (T concept : conceptList) { - Map conceptMap = new TreeMap<>(); - conceptMap.put(concept.getName(), concept); - toscaEntityMapList.add(conceptMap); - } - - return toscaEntityMapList; - } - - /** - * Return the contents of a list of concepts as map of concepts. - * - * @param conceptList the concept list - * @return the list of concept map - */ - private Map asConceptMap(List conceptList) { - Map conceptMap = new LinkedHashMap<>(); - for (T concept : conceptList) { - conceptMap.put(concept.getName(), concept); - } - - return conceptMap; - } - /** * Handle a PfModelRuntimeException on a list call. * diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/provider/SimpleToscaProvider.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/provider/SimpleToscaProvider.java index a3e18ca41..805f9dfed 100644 --- a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/provider/SimpleToscaProvider.java +++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/provider/SimpleToscaProvider.java @@ -133,13 +133,32 @@ public class SimpleToscaProvider { throws PfModelException { LOGGER.debug("->getDataTypes: name={}, version={}", name, version); - JpaToscaServiceTemplate serviceTemplate = getServiceTemplate(dao); + final JpaToscaServiceTemplate dbServiceTemplate = getServiceTemplate(dao); - if (!ToscaUtils.doDataTypesExist(serviceTemplate)) { + if (!ToscaUtils.doDataTypesExist(dbServiceTemplate)) { throw new PfModelRuntimeException(Response.Status.NOT_FOUND, "data types for " + name + ":" + version + DO_NOT_EXIST); } + JpaToscaServiceTemplate serviceTemplate = getCascadedDataTypes(dbServiceTemplate, name, version); + + LOGGER.debug("<-getDataTypes: name={}, version={}, serviceTemplate={}", name, version, serviceTemplate); + return serviceTemplate; + } + + /** + * Get the cascaded data types for a data type name and version. + * + * @param dbServiceTemplate the service template to search for the cascaded data types + * @param name the data type name we are searching for + * @param version the data type version we are searching for + * @return a service template containing the cascaded data types + * @throws PfModelException on errors getting the data types + */ + public JpaToscaServiceTemplate getCascadedDataTypes(@NonNull final JpaToscaServiceTemplate dbServiceTemplate, + final String name, final String version) throws PfModelException { + + JpaToscaServiceTemplate serviceTemplate = new JpaToscaServiceTemplate(dbServiceTemplate); serviceTemplate.setPolicyTypes(null); serviceTemplate.setTopologyTemplate(null); @@ -154,15 +173,13 @@ public class SimpleToscaProvider { Collection referencedDataTypeKeys = dataType.getReferencedDataTypes(); for (PfConceptKey referencedDataTypeKey : referencedDataTypeKeys) { - JpaToscaServiceTemplate dataTypeEntityTreeServiceTemplate = - getDataTypes(dao, referencedDataTypeKey.getName(), referencedDataTypeKey.getVersion()); + JpaToscaServiceTemplate dataTypeEntityTreeServiceTemplate = getCascadedDataTypes(dbServiceTemplate, + referencedDataTypeKey.getName(), referencedDataTypeKey.getVersion()); serviceTemplate = ToscaServiceTemplateUtils.addFragment(serviceTemplate, dataTypeEntityTreeServiceTemplate); } } - - LOGGER.debug("<-getDataTypes: name={}, version={}, serviceTemplate={}", name, version, serviceTemplate); return serviceTemplate; } @@ -277,16 +294,36 @@ public class SimpleToscaProvider { throws PfModelException { LOGGER.debug("->getPolicyTypes: name={}, version={}", name, version); - JpaToscaServiceTemplate serviceTemplate = getServiceTemplate(dao); - - serviceTemplate.setDataTypes(null); - serviceTemplate.setTopologyTemplate(null); + final JpaToscaServiceTemplate dbServiceTemplate = getServiceTemplate(dao); - if (!ToscaUtils.doPolicyTypesExist(serviceTemplate)) { + if (!ToscaUtils.doPolicyTypesExist(dbServiceTemplate)) { throw new PfModelRuntimeException(Response.Status.NOT_FOUND, "policy types for " + name + ":" + version + DO_NOT_EXIST); } + JpaToscaServiceTemplate serviceTemplate = getCascadedPolicyTypes(dbServiceTemplate, name, version); + + LOGGER.debug("<-getPolicyTypes: name={}, version={}, serviceTemplate={}", name, version, serviceTemplate); + return serviceTemplate; + } + + /** + * Get the cascaded policy types for a policy type name and version. + * + * @param dbServiceTemplate the service template to search for the cascaded policy types + * @param name the policy type name we are searching for + * @param version the policy type version we are searching for + * @return a service template containing the cascaded policy types + * @throws PfModelException on errors getting the policy types + */ + public JpaToscaServiceTemplate getCascadedPolicyTypes(final JpaToscaServiceTemplate dbServiceTemplate, + final String name, final String version) throws PfModelException { + + JpaToscaServiceTemplate serviceTemplate = new JpaToscaServiceTemplate(dbServiceTemplate); + + serviceTemplate.setDataTypes(null); + serviceTemplate.setTopologyTemplate(null); + ToscaUtils.getEntityTree(serviceTemplate.getPolicyTypes(), name, version); if (!ToscaUtils.doPolicyTypesExist(serviceTemplate)) { @@ -301,8 +338,8 @@ public class SimpleToscaProvider { Collection referencedDataTypeKeys = policyType.getReferencedDataTypes(); for (PfConceptKey referencedDataTypeKey : referencedDataTypeKeys) { - JpaToscaServiceTemplate dataTypeEntityTreeServiceTemplate = - getDataTypes(dao, referencedDataTypeKey.getName(), referencedDataTypeKey.getVersion()); + JpaToscaServiceTemplate dataTypeEntityTreeServiceTemplate = getCascadedDataTypes(dbServiceTemplate, + referencedDataTypeKey.getName(), referencedDataTypeKey.getVersion()); dataTypeServiceTemplate = ToscaServiceTemplateUtils.addFragment(dataTypeServiceTemplate, dataTypeEntityTreeServiceTemplate); @@ -310,8 +347,6 @@ public class SimpleToscaProvider { } serviceTemplate = ToscaServiceTemplateUtils.addFragment(serviceTemplate, dataTypeServiceTemplate); - - LOGGER.debug("<-getPolicyTypes: name={}, version={}, serviceTemplate={}", name, version, serviceTemplate); return serviceTemplate; } @@ -438,15 +473,33 @@ public class SimpleToscaProvider { JpaToscaServiceTemplate dbServiceTemplate = getServiceTemplate(dao); - JpaToscaServiceTemplate serviceTemplate = new JpaToscaServiceTemplate(dbServiceTemplate); - serviceTemplate.setDataTypes(new JpaToscaDataTypes()); - serviceTemplate.setPolicyTypes(new JpaToscaPolicyTypes()); - - if (!ToscaUtils.doPoliciesExist(serviceTemplate)) { + if (!ToscaUtils.doPoliciesExist(dbServiceTemplate)) { throw new PfModelRuntimeException(Response.Status.NOT_FOUND, "policies for " + name + ":" + version + DO_NOT_EXIST); } + JpaToscaServiceTemplate returnServiceTemplate = getCascadedPolicies(dbServiceTemplate, name, version); + + LOGGER.debug("<-getPolicies: name={}, version={}, serviceTemplate={}", name, version, returnServiceTemplate); + return returnServiceTemplate; + } + + /** + * Get the cascaded policies for a policy name and version. + * + * @param dbServiceTemplate the service template to search for the cascaded policy + * @param name the policy name we are searching for + * @param version the policy version we are searching for + * @return a service template containing the cascaded policy + * @throws PfModelException on errors getting the policy + */ + public JpaToscaServiceTemplate getCascadedPolicies(final JpaToscaServiceTemplate dbServiceTemplate, + final String name, final String version) throws PfModelException { + + JpaToscaServiceTemplate serviceTemplate = new JpaToscaServiceTemplate(dbServiceTemplate); + serviceTemplate.setDataTypes(new JpaToscaDataTypes()); + serviceTemplate.setPolicyTypes(new JpaToscaPolicyTypes()); + ToscaUtils.getEntityTree(serviceTemplate.getTopologyTemplate().getPolicies(), name, version); if (!ToscaUtils.doPoliciesExist(serviceTemplate)) { @@ -458,15 +511,14 @@ public class SimpleToscaProvider { returnServiceTemplate.getTopologyTemplate().setPolicies(new JpaToscaPolicies()); for (JpaToscaPolicy policy : serviceTemplate.getTopologyTemplate().getPolicies().getConceptMap().values()) { - JpaToscaServiceTemplate referencedEntitiesServiceTemplate = - getPolicyTypes(dao, policy.getType().getName(), policy.getType().getVersion()); + JpaToscaServiceTemplate referencedEntitiesServiceTemplate = getCascadedPolicyTypes(dbServiceTemplate, + policy.getType().getName(), policy.getType().getVersion()); returnServiceTemplate.getTopologyTemplate().getPolicies().getConceptMap().put(policy.getKey(), policy); returnServiceTemplate = ToscaServiceTemplateUtils.addFragment(returnServiceTemplate, referencedEntitiesServiceTemplate); } - LOGGER.debug("<-getPolicies: name={}, version={}, serviceTemplate={}", name, version, returnServiceTemplate); return returnServiceTemplate; } -- 2.16.6