From: Christophe Closset Date: Wed, 18 Nov 2020 09:15:49 +0000 (+0000) Subject: Merge "Update clamp to 5.1.5" into guilin X-Git-Url: https://gerrit.onap.org/r/gitweb?a=commitdiff_plain;h=refs%2Fheads%2Fguilin;hp=b30c05e46a4bd7761620c481092c2377dcc108f6;p=clamp.git Merge "Update clamp to 5.1.5" into guilin --- diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 175febc4..28d51170 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -6,6 +6,53 @@ Release Notes ============= +Version: 5.1.0 (Guilin) +-------------------------- + +:Release Date: 2020-11-19 + +**New Features** + +The Guilin release of the Control Loop Automation Management Platform (CLAMP). + +The main goal of the Guilin release was to: + + - Complete integration to CDS for Actor/Action selection. + - SECCOM Perform Software Composition Analysis - Vulnerability tables (TSC must have). + - SECCOM Password removal from OOM HELM charts (TSC must have) - implementation of certinInitializer to get AAF certificates at oom deployment time. + +**Bug Fixes** + + - The full list of implemented user stories and epics is available on `CLAMP R7 - M1 release planning `_ + - The full list of issues(bugs) solved, is available on `CLAMP R7 - Guilin list of solved issues(bugs) `_ + +**Known Issues** + +**Security Notes** + +*Fixed Security Issues* + +*Known Security Issues* + +*Known Vulnerabilities in Used Modules* + +CLAMP code has been formally scanned during build time using NexusIQ and all Critical vulnerabilities have been addressed, items that remain open have been assessed for risk and actions to be taken in future release. + +Quick Links: + - `CLAMP project page `_ + + - `Passing Badge information for CLAMP `_ + +**Upgrade Notes** + + - The Upgrade strategy for Guilin can be found here:``_ + - New Docker Containers are available. the list of containers composing this release are below: + - clamp-backend: nexus3.onap.org:10001/onap/clamp-backend 5.1.5 + - clamp-frontend: nexus3.onap.org:10001/onap/clamp-frontend 5.1.5 + - clamp-dash-es: nexus3.onap.org:10001/onap/clamp-dashboard-elasticsearch 5.0.4 + - clamp-dash-kibana: nexus3.onap.org:10001/onap/clamp-dashboard-kibana 5.0.4 + - clamp-dash-logstash: nexus3.onap.org:10001/onap/clamp-dashboard-logstash 5.0.4 + Version: 5.0.7 (Frankfurt maintenance release tag 6.0.0) -------------------------------------------------------- diff --git a/src/main/java/org/onap/clamp/clds/client/PolicyEngineServices.java b/src/main/java/org/onap/clamp/clds/client/PolicyEngineServices.java index 260bd1e4..c75d733a 100644 --- a/src/main/java/org/onap/clamp/clds/client/PolicyEngineServices.java +++ b/src/main/java/org/onap/clamp/clds/client/PolicyEngineServices.java @@ -87,20 +87,29 @@ public class PolicyEngineServices { /** * This method query Policy engine and create a PolicyModel object with type and version. + * If the policy already exist in the db it returns the existing one. * * @param policyType The policyType id * @param policyVersion The policy version of that type - * @return A PolicyModel created from policyEngine data + * @return A PolicyModel created from policyEngine data or null if nothing is found on policyEngine */ public PolicyModel createPolicyModelFromPolicyEngine(String policyType, String policyVersion) { - if (!policyModelsService.existsById( - new PolicyModelId(policyType, policyVersion))) { - return policyModelsService.savePolicyModelInNewTransaction( - new PolicyModel(policyType, this.downloadOnePolicy(policyType, policyVersion), policyVersion)); + PolicyModel policyModelFound = policyModelsService.getPolicyModel(policyType, policyVersion); + if (policyModelFound == null) { + String policyTosca = this.downloadOnePolicy(policyType, policyVersion); + if (policyTosca != null && !policyTosca.isEmpty()) { + return policyModelsService.savePolicyModelInNewTransaction( + new PolicyModel(policyType, policyTosca, policyVersion)); + } else { + logger.error("Policy not found in the Policy Engine, returning null: " + policyType + + "/" + policyVersion); + return null; + } + } else { + logger.info("Skipping policy model download as it exists already in the database " + policyType + + "/" + policyVersion); + return policyModelFound; } - logger.info("Skipping policy model download as it exists already in the database " + policyType - + "/" + policyVersion); - return null; } /** @@ -158,10 +167,17 @@ public class PolicyEngineServices { options.setPrettyFlow(true); options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); Yaml yamlParser = new Yaml(options); - return yamlParser.dump((Map) yamlParser.load(callCamelRoute( + String responseBody = callCamelRoute( ExchangeBuilder.anExchange(camelContext).withProperty("policyModelName", policyType) .withProperty("policyModelVersion", policyVersion).build(), "direct:get-policy-model", - "Get one policy"))); + "Get one policy"); + + if (responseBody == null || responseBody.isEmpty()) { + logger.warn("getPolicyModel returned by policy engine could not be decoded, as it's null or empty"); + return null; + } + + return yamlParser.dump((Map) yamlParser.load(responseBody)); } /** @@ -196,8 +212,7 @@ public class PolicyEngineServices { Exchange exchangeResponse = camelContext.createProducerTemplate().send(camelFlow, exchange); if (Integer.valueOf(200).equals(exchangeResponse.getIn().getHeader("CamelHttpResponseCode"))) { return (String) exchangeResponse.getIn().getBody(); - } - else { + } else { logger.info(logMsg + " query " + retryInterval + "ms before retrying ..."); // wait for a while and try to connect to DCAE again try { diff --git a/src/main/java/org/onap/clamp/loop/CsarInstaller.java b/src/main/java/org/onap/clamp/loop/CsarInstaller.java index 67c7ce5c..6752a1a3 100644 --- a/src/main/java/org/onap/clamp/loop/CsarInstaller.java +++ b/src/main/java/org/onap/clamp/loop/CsarInstaller.java @@ -152,7 +152,8 @@ public class CsarInstaller { private LoopTemplate createLoopTemplateFromBlueprint(CsarHandler csar, BlueprintArtifact blueprintArtifact, Service service) - throws IOException, ParseException, InterruptedException, BlueprintParserException { + throws IOException, ParseException, InterruptedException, BlueprintParserException, + SdcArtifactInstallerException { LoopTemplate newLoopTemplate = new LoopTemplate(); newLoopTemplate.setBlueprint(blueprintArtifact.getDcaeBlueprint()); newLoopTemplate.setName(LoopTemplate.generateLoopTemplateName(csar.getSdcNotification().getServiceName(), @@ -165,32 +166,35 @@ public class CsarInstaller { microServicesChain = BlueprintParser.fallbackToOneMicroService(); } newLoopTemplate.setModelService(service); - newLoopTemplate.addLoopElementModels(createMicroServiceModels(microServicesChain)); + newLoopTemplate.addLoopElementModels(createMicroServiceModels(blueprintArtifact, microServicesChain)); newLoopTemplate.setMaximumInstancesAllowed(0); DcaeInventoryResponse dcaeResponse = queryDcaeToGetServiceTypeId(blueprintArtifact); newLoopTemplate.setDcaeBlueprintId(dcaeResponse.getTypeId()); return newLoopTemplate; } - private HashSet createMicroServiceModels(List microServicesChain) - throws InterruptedException { + private HashSet createMicroServiceModels(BlueprintArtifact blueprintArtifact, + List microServicesChain) + throws SdcArtifactInstallerException { HashSet newSet = new HashSet<>(); for (BlueprintMicroService microService : microServicesChain) { LoopElementModel loopElementModel = new LoopElementModel(microService.getModelType(), LoopElementModel.MICRO_SERVICE_TYPE, null); newSet.add(loopElementModel); - loopElementModel.addPolicyModel(getPolicyModel(microService)); + PolicyModel newPolicyModel = policyEngineServices.createPolicyModelFromPolicyEngine(microService); + if (newPolicyModel != null) { + loopElementModel.addPolicyModel(newPolicyModel); + } else { + throw new SdcArtifactInstallerException( + "Unable to find the policy specified in the blueprint " + + blueprintArtifact.getBlueprintArtifactName() + ") on the Policy Engine:" + + microService.getModelType() + "/" + microService.getModelVersion()); + } } return newSet; } - private PolicyModel getPolicyModel(BlueprintMicroService microService) throws InterruptedException { - return policyModelsRepository - .findById(new PolicyModelId(microService.getModelType(), microService.getModelVersion())) - .orElse(policyEngineServices.createPolicyModelFromPolicyEngine(microService)); - } - /** * Get the service blueprint Id in the Dcae inventory using the SDC UUID. * diff --git a/src/main/java/org/onap/clamp/loop/template/PolicyModelsService.java b/src/main/java/org/onap/clamp/loop/template/PolicyModelsService.java index a1b8f7cf..17cf5c1c 100644 --- a/src/main/java/org/onap/clamp/loop/template/PolicyModelsService.java +++ b/src/main/java/org/onap/clamp/loop/template/PolicyModelsService.java @@ -100,7 +100,7 @@ public class PolicyModelsService { public PolicyModel updatePolicyModelTosca(String policyModelType, String policyModelVersion, String policyModelTosca) { JsonObject jsonObject = toscaYamlToJsonConvertor.validateAndConvertToJson(policyModelTosca); - PolicyModel thePolicyModel = getPolicyModelByType(policyModelType, policyModelVersion); + PolicyModel thePolicyModel = getPolicyModel(policyModelType, policyModelVersion); thePolicyModel.setPolicyAcronym(toscaYamlToJsonConvertor.getValueFromMetadata(jsonObject, ToscaSchemaConstants.METADATA_ACRONYM)); thePolicyModel.setPolicyModelTosca(policyModelTosca); @@ -123,10 +123,6 @@ public class PolicyModelsService { return policyModelsRepository.findByPolicyModelType(type); } - public PolicyModel getPolicyModelByType(String type, String version) { - return policyModelsRepository.findById(new PolicyModelId(type, version)).orElse(null); - } - /** * Retrieves the Tosca model Yaml string. * diff --git a/src/test/java/org/onap/clamp/loop/CsarInstallerItCase.java b/src/test/java/org/onap/clamp/loop/CsarInstallerItCase.java index aa8054c4..f986ca93 100644 --- a/src/test/java/org/onap/clamp/loop/CsarInstallerItCase.java +++ b/src/test/java/org/onap/clamp/loop/CsarInstallerItCase.java @@ -112,6 +112,48 @@ public class CsarInstallerItCase { return blueprintArtifact; } + private CsarHandler buildBadFakeCsarHandler(String generatedName, String csarFileName) throws IOException, + SdcToscaParserException { + + // Build a Bad csar because the blueprint contains a link to a microservice that does not exist in the emulator + // Create fake notification + INotificationData notificationData = Mockito.mock(INotificationData.class); + Mockito.when(notificationData.getServiceVersion()).thenReturn("1.0"); + // Create fake resource in notification + CsarHandler csarHandler = Mockito.mock(CsarHandler.class); + List listResources = new ArrayList<>(); + Mockito.when(notificationData.getResources()).thenReturn(listResources); + Map blueprintMap = new HashMap<>(); + Mockito.when(csarHandler.getMapOfBlueprints()).thenReturn(blueprintMap); + // Create fake blueprint artifact 1 on resource1 + BlueprintArtifact blueprintArtifact = buildFakeBuildprintArtifact(RESOURCE_INSTANCE_NAME_RESOURCE1, + INVARIANT_RESOURCE1_UUID, "example/sdc/blueprint-dcae/tca-guilin.yaml", "tca-guilin.yaml", + INVARIANT_SERVICE_UUID); + listResources.add(blueprintArtifact.getResourceAttached()); + blueprintMap.put(blueprintArtifact.getBlueprintArtifactName(), blueprintArtifact); + + // Build fake csarhandler + Mockito.when(csarHandler.getSdcNotification()).thenReturn(notificationData); + // Build fake csar Helper + ISdcCsarHelper csarHelper = Mockito.mock(ISdcCsarHelper.class); + Metadata data = Mockito.mock(Metadata.class); + Mockito.when(data.getValue("name")).thenReturn(generatedName); + Mockito.when(notificationData.getServiceName()).thenReturn(generatedName); + Mockito.when(csarHelper.getServiceMetadata()).thenReturn(data); + + // Create helper based on real csar to test policy yaml and global properties + // set + SdcToscaParserFactory factory = SdcToscaParserFactory.getInstance(); + String path = Thread.currentThread().getContextClassLoader().getResource(csarFileName).getFile(); + ISdcCsarHelper sdcHelper = factory.getSdcCsarHelper(path); + Mockito.when(csarHandler.getSdcCsarHelper()).thenReturn(sdcHelper); + + // Mockito.when(csarHandler.getSdcCsarHelper()).thenReturn(csarHelper); + Mockito.when(csarHandler.getPolicyModelYaml()) + .thenReturn(Optional.ofNullable(ResourceFileUtils.getResourceAsString("tosca/tosca_example.yaml"))); + return csarHandler; + } + private CsarHandler buildFakeCsarHandler(String generatedName, String csarFileName) throws IOException, SdcToscaParserException { // Create fake notification @@ -203,14 +245,25 @@ public class CsarInstallerItCase { assertThat(csarInstaller.isCsarAlreadyDeployed(csarHandler)).isTrue(); } + @Test(expected = SdcArtifactInstallerException.class) + @Transactional + public void testInstallTheBadCsarTca() + throws IOException, SdcToscaParserException, InterruptedException, BlueprintParserException, + SdcArtifactInstallerException { + // This test validates that the blueprint is well rejected because the blueprint contains a link + // to a policy that does not exist on the policy engine emulator. + String generatedName = RandomStringUtils.randomAlphanumeric(5); + csarInstaller.installTheCsar(buildBadFakeCsarHandler(generatedName, CSAR_ARTIFACT_NAME_NO_CDS)); + } + @Test @Transactional @Commit public void testInstallTheCsarTca() throws SdcArtifactInstallerException, SdcToscaParserException, CsarHandlerException, IOException, JSONException, InterruptedException, BlueprintParserException { String generatedName = RandomStringUtils.randomAlphanumeric(5); - CsarHandler csar = buildFakeCsarHandler(generatedName, CSAR_ARTIFACT_NAME_CDS); - csarInstaller.installTheCsar(csar); + csarInstaller.installTheCsar(buildFakeCsarHandler(generatedName, CSAR_ARTIFACT_NAME_CDS)); + assertThat(serviceRepository.existsById("63cac700-ab9a-4115-a74f-7eac85e3fce0")).isTrue(); // We should have CDS info assertThat(serviceRepository.findById("63cac700-ab9a-4115-a74f-7eac85e3fce0").get().getResourceByType("VF") diff --git a/src/test/resources/example/sdc/blueprint-dcae/tca-guilin.yaml b/src/test/resources/example/sdc/blueprint-dcae/tca-guilin.yaml new file mode 100644 index 00000000..e7d967a2 --- /dev/null +++ b/src/test/resources/example/sdc/blueprint-dcae/tca-guilin.yaml @@ -0,0 +1,141 @@ +# ============LICENSE_START==================================================== +# ============================================================================= +# Copyright (C) 2019-2020 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== +#k8s-tca-gen2-v3.yaml + +tosca_definitions_version: cloudify_dsl_1_3 +imports: + - https://www.getcloudify.org/spec/cloudify/4.5.5/types.yaml + - plugin:k8splugin?version=3.4.2 + - plugin:clamppolicyplugin?version=1.1.0 +inputs: + service_name: + type: string + default: 'dcae-tcagen2' + log_directory: + type: string + default: "/opt/logs/dcae-analytics-tca" + replicas: + type: integer + description: number of instances + default: 1 + spring.data.mongodb.uri: + type: string + default: "mongodb://dcae-mongohost/dcae-tcagen2" + tag_version: + type: string + default: "nexus3.onap.org:10001/onap/org.onap.dcaegen2.analytics.tca-gen2.dcae-analytics-tca-web:1.2.1" + tca.aai.password: + type: string + default: "DCAE" + tca.aai.url: + type: string + default: "http://aai.onap.svc.cluster.local" + tca.aai.username: + type: string + default: "DCAE" + tca_handle_in_subscribe_url: + type: string + default: "http://message-router.onap.svc.cluster.local:3904/events/unauthenticated.VES_MEASUREMENT_OUTPUT" + tca_handle_out_publish_url: + type: string + default: "http://message-router.onap.svc.cluster.local:3904/events/unauthenticated.DCAE_CL_OUTPUT" + tca_consumer_group: + type: string + default: "cg1" + policy_model_id: + type: string + default: "onap.policies.monitoring.tcagen2" + policy_id: + type: string + default: "onap.restart.tca" +node_templates: + docker.tca: + type: dcae.nodes.ContainerizedServiceComponent + relationships: + - target: tcagen2_policy + type: cloudify.relationships.depends_on + interfaces: + cloudify.interfaces.lifecycle: + start: + inputs: + ports: + - concat: ["9091:", "0"] + properties: + application_config: + service_calls: [] + streams_publishes: + tca_handle_out: + dmaap_info: + topic_url: + get_input: tca_handle_out_publish_url + type: message_router + streams_subscribes: + tca_handle_in: + dmaap_info: + topic_url: + get_input: tca_handle_in_subscribe_url + type: message_router + spring.data.mongodb.uri: + get_input: spring.data.mongodb.uri + streams_subscribes.tca_handle_in.consumer_group: + get_input: tca_consumer_group + streams_subscribes.tca_handle_in.consumer_ids[0]: c0 + streams_subscribes.tca_handle_in.consumer_ids[1]: c1 + streams_subscribes.tca_handle_in.message_limit: 50000 + streams_subscribes.tca_handle_in.polling.auto_adjusting.max: 60000 + streams_subscribes.tca_handle_in.polling.auto_adjusting.min: 30000 + streams_subscribes.tca_handle_in.polling.auto_adjusting.step_down: 30000 + streams_subscribes.tca_handle_in.polling.auto_adjusting.step_up: 10000 + streams_subscribes.tca_handle_in.polling.fixed_rate: 0 + streams_subscribes.tca_handle_in.timeout: -1 + tca.aai.enable_enrichment: true + tca.aai.generic_vnf_path: aai/v11/network/generic-vnfs/generic-vnf + tca.aai.node_query_path: aai/v11/search/nodes-query + tca.aai.password: + get_input: tca.aai.password + tca.aai.url: + get_input: tca.aai.url + tca.aai.username: + get_input: tca.aai.username + tca.policy: '{"domain":"measurementsForVfScaling","metricsPerEventName":[{"eventName":"vFirewallBroadcastPackets","controlLoopSchemaType":"VM","policyScope":"DCAE","policyName":"DCAE.Config_tca-hi-lo","policyVersion":"v0.0.1","thresholds":[{"closedLoopControlName":"ControlLoop-vFirewall-d0a1dfc6-94f5-4fd4-a5b5-4630b438850a","version":"1.0.2","fieldPath":"$.event.measurementsForVfScalingFields.vNicPerformanceArray[*].receivedTotalPacketsDelta","thresholdValue":300,"direction":"LESS_OR_EQUAL","severity":"MAJOR","closedLoopEventStatus":"ONSET"},{"closedLoopControlName":"ControlLoop-vFirewall-d0a1dfc6-94f5-4fd4-a5b5-4630b438850a","version":"1.0.2","fieldPath":"$.event.measurementsForVfScalingFields.vNicPerformanceArray[*].receivedTotalPacketsDelta","thresholdValue":700,"direction":"GREATER_OR_EQUAL","severity":"CRITICAL","closedLoopEventStatus":"ONSET"}]},{"eventName":"vLoadBalancer","controlLoopSchemaType":"VM","policyScope":"DCAE","policyName":"DCAE.Config_tca-hi-lo","policyVersion":"v0.0.1","thresholds":[{"closedLoopControlName":"ControlLoop-vDNS-6f37f56d-a87d-4b85-b6a9-cc953cf779b3","version":"1.0.2","fieldPath":"$.event.measurementsForVfScalingFields.vNicPerformanceArray[*].receivedTotalPacketsDelta","thresholdValue":300,"direction":"GREATER_OR_EQUAL","severity":"CRITICAL","closedLoopEventStatus":"ONSET"}]},{"eventName":"Measurement_vGMUX","controlLoopSchemaType":"VNF","policyScope":"DCAE","policyName":"DCAE.Config_tca-hi-lo","policyVersion":"v0.0.1","thresholds":[{"closedLoopControlName":"ControlLoop-vCPE-48f0c2c3-a172-4192-9ae3-052274181b6e","version":"1.0.2","fieldPath":"$.event.measurementsForVfScalingFields.additionalMeasurements[*].arrayOfFields[0].value","thresholdValue":0,"direction":"EQUAL","severity":"MAJOR","closedLoopEventStatus":"ABATED"},{"closedLoopControlName":"ControlLoop-vCPE-48f0c2c3-a172-4192-9ae3-052274181b6e","version":"1.0.2","fieldPath":"$.event.measurementsForVfScalingFields.additionalMeasurements[*].arrayOfFields[0].value","thresholdValue":0,"direction":"GREATER","severity":"CRITICAL","closedLoopEventStatus":"ONSET"}]}]}' + tca.processing_batch_size: 10000 + tca.enable_abatement: true + tca.enable_ecomp_logging: true + docker_config: + healthcheck: + endpoint: /actuator/health + interval: 30s + timeout: 10s + type: http + image: + get_input: tag_version + log_info: + log_directory: + get_input: log_directory + tls_info: + use_tls: true + cert_directory: '/etc/tca-gen2/ssl' + replicas: + get_input: replicas + service_component_type: { get_input: service_name } + tcagen2_policy: + type: clamp.nodes.policy + properties: + policy_id: + get_input: policy_id + policy_model_id: + get_input: policy_model_id