Merge "Update clamp to 5.1.5" into guilin

diff --git a/docs/release-notes.rst b/docs/release-notes.rst
index 175febc..28d5117 100644 (file)
--- a/docs/release-notes.rst
+++ b/docs/release-notes.rst
@@ -6,6 +6,53 @@
 Release Notes
 =============
 
+Version: 5.1.0 (Guilin)
+--------------------------
+
+:Release Date: 2020-11-19
+
+**New Features**
+
+The Guilin release of the Control Loop Automation Management Platform (CLAMP).
+
+The main goal of the Guilin release was to:
+
+    - Complete integration to CDS for Actor/Action selection.
+    - SECCOM Perform Software Composition Analysis - Vulnerability tables (TSC must have).
+    - SECCOM Password removal from OOM HELM charts (TSC must have) - implementation of certinInitializer to get AAF certificates at oom deployment time.
+
+**Bug Fixes**
+
+       - The full list of implemented user stories and epics is available on `CLAMP R7 - M1 release planning <https://wiki.onap.org/display/DW/CLAMP+R7+-+M1+Release+Planning>`_
+       - The full list of issues(bugs) solved, is available on `CLAMP R7 - Guilin list of solved issues(bugs) <https://wiki.onap.org/pages/viewpage.action?pageId=93000705>`_
+
+**Known Issues**
+
+**Security Notes**
+
+*Fixed Security Issues*
+
+*Known Security Issues*
+
+*Known Vulnerabilities in Used Modules*
+
+CLAMP code has been formally scanned during build time using NexusIQ and all Critical vulnerabilities have been addressed, items that remain open have been assessed for risk and actions to be taken in future release.
+
+Quick Links:
+       - `CLAMP project page <https://wiki.onap.org/display/DW/CLAMP+Project>`_
+
+       - `Passing Badge information for CLAMP <https://bestpractices.coreinfrastructure.org/en/projects/1197>`_
+
+**Upgrade Notes**
+
+    - The Upgrade strategy for Guilin can be found here:`<https://wiki.onap.org/display/DW/Frankfurt+CLAMP+Container+upgrade+strategy>`_
+    - New Docker Containers are available. the list of containers composing this release are below:
+       - clamp-backend: nexus3.onap.org:10001/onap/clamp-backend 5.1.5
+       - clamp-frontend: nexus3.onap.org:10001/onap/clamp-frontend 5.1.5
+       - clamp-dash-es: nexus3.onap.org:10001/onap/clamp-dashboard-elasticsearch 5.0.4
+       - clamp-dash-kibana: nexus3.onap.org:10001/onap/clamp-dashboard-kibana 5.0.4
+       - clamp-dash-logstash: nexus3.onap.org:10001/onap/clamp-dashboard-logstash 5.0.4
+
 Version: 5.0.7 (Frankfurt maintenance release tag 6.0.0)
 --------------------------------------------------------
 

diff --git a/src/main/java/org/onap/clamp/clds/client/PolicyEngineServices.java b/src/main/java/org/onap/clamp/clds/client/PolicyEngineServices.java
index 260bd1e..c75d733 100644 (file)
--- a/src/main/java/org/onap/clamp/clds/client/PolicyEngineServices.java
+++ b/src/main/java/org/onap/clamp/clds/client/PolicyEngineServices.java
@@ -87,20 +87,29 @@ public class PolicyEngineServices {
 
     /**
      * This method query Policy engine and create a PolicyModel object with type and version.
+     * If the policy already exist in the db it returns the existing one.
      *
      * @param policyType    The policyType id
      * @param policyVersion The policy version of that type
-     * @return A PolicyModel created from policyEngine data
+     * @return A PolicyModel created from policyEngine data or null if nothing is found on policyEngine
      */
     public PolicyModel createPolicyModelFromPolicyEngine(String policyType, String policyVersion) {
-        if (!policyModelsService.existsById(
-                new PolicyModelId(policyType, policyVersion))) {
-            return policyModelsService.savePolicyModelInNewTransaction(
-                    new PolicyModel(policyType, this.downloadOnePolicy(policyType, policyVersion), policyVersion));
+        PolicyModel policyModelFound = policyModelsService.getPolicyModel(policyType, policyVersion);
+        if (policyModelFound == null) {
+            String policyTosca = this.downloadOnePolicy(policyType, policyVersion);
+            if (policyTosca != null && !policyTosca.isEmpty()) {
+                return policyModelsService.savePolicyModelInNewTransaction(
+                        new PolicyModel(policyType, policyTosca, policyVersion));
+            } else {
+                logger.error("Policy not found in the Policy Engine, returning null: " + policyType
+                        + "/" + policyVersion);
+                return null;
+            }
+        } else {
+            logger.info("Skipping policy model download as it exists already in the database " + policyType
+                    + "/" + policyVersion);
+            return policyModelFound;
         }
-        logger.info("Skipping policy model download as it exists already in the database " + policyType
-                + "/" + policyVersion);
-        return null;
     }
 
     /**
@@ -158,10 +167,17 @@ public class PolicyEngineServices {
         options.setPrettyFlow(true);
         options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK);
         Yaml yamlParser = new Yaml(options);
-        return yamlParser.dump((Map<String, Object>) yamlParser.load(callCamelRoute(
+        String responseBody = callCamelRoute(
                 ExchangeBuilder.anExchange(camelContext).withProperty("policyModelName", policyType)
                         .withProperty("policyModelVersion", policyVersion).build(), "direct:get-policy-model",
-                "Get one policy")));
+                "Get one policy");
+
+        if (responseBody == null || responseBody.isEmpty()) {
+            logger.warn("getPolicyModel returned by policy engine could not be decoded, as it's null or empty");
+            return null;
+        }
+
+        return yamlParser.dump((Map<String, Object>) yamlParser.load(responseBody));
     }
 
     /**
@@ -196,8 +212,7 @@ public class PolicyEngineServices {
             Exchange exchangeResponse = camelContext.createProducerTemplate().send(camelFlow, exchange);
             if (Integer.valueOf(200).equals(exchangeResponse.getIn().getHeader("CamelHttpResponseCode"))) {
                 return (String) exchangeResponse.getIn().getBody();
-            }
-            else {
+            } else {
                 logger.info(logMsg + " query " + retryInterval + "ms before retrying ...");
                 // wait for a while and try to connect to DCAE again
                 try {

diff --git a/src/main/java/org/onap/clamp/loop/CsarInstaller.java b/src/main/java/org/onap/clamp/loop/CsarInstaller.java
index 67c7ce5..6752a1a 100644 (file)
--- a/src/main/java/org/onap/clamp/loop/CsarInstaller.java
+++ b/src/main/java/org/onap/clamp/loop/CsarInstaller.java
@@ -152,7 +152,8 @@ public class CsarInstaller {
 
     private LoopTemplate createLoopTemplateFromBlueprint(CsarHandler csar, BlueprintArtifact blueprintArtifact,
                                                          Service service)
-            throws IOException, ParseException, InterruptedException, BlueprintParserException {
+            throws IOException, ParseException, InterruptedException, BlueprintParserException,
+            SdcArtifactInstallerException {
         LoopTemplate newLoopTemplate = new LoopTemplate();
         newLoopTemplate.setBlueprint(blueprintArtifact.getDcaeBlueprint());
         newLoopTemplate.setName(LoopTemplate.generateLoopTemplateName(csar.getSdcNotification().getServiceName(),
@@ -165,32 +166,35 @@ public class CsarInstaller {
             microServicesChain = BlueprintParser.fallbackToOneMicroService();
         }
         newLoopTemplate.setModelService(service);
-        newLoopTemplate.addLoopElementModels(createMicroServiceModels(microServicesChain));
+        newLoopTemplate.addLoopElementModels(createMicroServiceModels(blueprintArtifact, microServicesChain));
         newLoopTemplate.setMaximumInstancesAllowed(0);
         DcaeInventoryResponse dcaeResponse = queryDcaeToGetServiceTypeId(blueprintArtifact);
         newLoopTemplate.setDcaeBlueprintId(dcaeResponse.getTypeId());
         return newLoopTemplate;
     }
 
-    private HashSet<LoopElementModel> createMicroServiceModels(List<BlueprintMicroService> microServicesChain)
-            throws InterruptedException {
+    private HashSet<LoopElementModel> createMicroServiceModels(BlueprintArtifact blueprintArtifact,
+                                                               List<BlueprintMicroService> microServicesChain)
+            throws SdcArtifactInstallerException {
         HashSet<LoopElementModel> newSet = new HashSet<>();
         for (BlueprintMicroService microService : microServicesChain) {
             LoopElementModel loopElementModel =
                     new LoopElementModel(microService.getModelType(), LoopElementModel.MICRO_SERVICE_TYPE,
                             null);
             newSet.add(loopElementModel);
-            loopElementModel.addPolicyModel(getPolicyModel(microService));
+            PolicyModel newPolicyModel = policyEngineServices.createPolicyModelFromPolicyEngine(microService);
+            if (newPolicyModel != null) {
+                loopElementModel.addPolicyModel(newPolicyModel);
+            } else {
+                throw new SdcArtifactInstallerException(
+                        "Unable to find the policy specified in the blueprint " +
+                                blueprintArtifact.getBlueprintArtifactName() + ") on the Policy Engine:" +
+                                microService.getModelType() + "/" + microService.getModelVersion());
+            }
         }
         return newSet;
     }
 
-    private PolicyModel getPolicyModel(BlueprintMicroService microService) throws InterruptedException {
-        return policyModelsRepository
-                .findById(new PolicyModelId(microService.getModelType(), microService.getModelVersion()))
-                .orElse(policyEngineServices.createPolicyModelFromPolicyEngine(microService));
-    }
-
     /**
      * Get the service blueprint Id in the Dcae inventory using the SDC UUID.
      *

diff --git a/src/main/java/org/onap/clamp/loop/template/PolicyModelsService.java b/src/main/java/org/onap/clamp/loop/template/PolicyModelsService.java
index a1b8f7c..17cf5c1 100644 (file)
--- a/src/main/java/org/onap/clamp/loop/template/PolicyModelsService.java
+++ b/src/main/java/org/onap/clamp/loop/template/PolicyModelsService.java
@@ -100,7 +100,7 @@ public class PolicyModelsService {
     public PolicyModel updatePolicyModelTosca(String policyModelType, String policyModelVersion,
         String policyModelTosca) {
         JsonObject jsonObject = toscaYamlToJsonConvertor.validateAndConvertToJson(policyModelTosca);
-        PolicyModel thePolicyModel = getPolicyModelByType(policyModelType, policyModelVersion);
+        PolicyModel thePolicyModel = getPolicyModel(policyModelType, policyModelVersion);
         thePolicyModel.setPolicyAcronym(toscaYamlToJsonConvertor.getValueFromMetadata(jsonObject,
             ToscaSchemaConstants.METADATA_ACRONYM));
         thePolicyModel.setPolicyModelTosca(policyModelTosca);
@@ -123,10 +123,6 @@ public class PolicyModelsService {
         return policyModelsRepository.findByPolicyModelType(type);
     }
 
-    public PolicyModel getPolicyModelByType(String type, String version) {
-        return policyModelsRepository.findById(new PolicyModelId(type, version)).orElse(null);
-    }
-
     /**
      * Retrieves the Tosca model Yaml string.
      *

diff --git a/src/test/java/org/onap/clamp/loop/CsarInstallerItCase.java b/src/test/java/org/onap/clamp/loop/CsarInstallerItCase.java
index aa8054c..f986ca9 100644 (file)
--- a/src/test/java/org/onap/clamp/loop/CsarInstallerItCase.java
+++ b/src/test/java/org/onap/clamp/loop/CsarInstallerItCase.java
@@ -112,6 +112,48 @@ public class CsarInstallerItCase {
         return blueprintArtifact;
     }
 
+    private CsarHandler buildBadFakeCsarHandler(String generatedName, String csarFileName) throws IOException,
+            SdcToscaParserException {
+
+        // Build a Bad csar because the blueprint contains a link to a microservice that does not exist in the emulator
+        // Create fake notification
+        INotificationData notificationData = Mockito.mock(INotificationData.class);
+        Mockito.when(notificationData.getServiceVersion()).thenReturn("1.0");
+        // Create fake resource in notification
+        CsarHandler csarHandler = Mockito.mock(CsarHandler.class);
+        List<IResourceInstance> listResources = new ArrayList<>();
+        Mockito.when(notificationData.getResources()).thenReturn(listResources);
+        Map<String, BlueprintArtifact> blueprintMap = new HashMap<>();
+        Mockito.when(csarHandler.getMapOfBlueprints()).thenReturn(blueprintMap);
+        // Create fake blueprint artifact 1 on resource1
+        BlueprintArtifact blueprintArtifact = buildFakeBuildprintArtifact(RESOURCE_INSTANCE_NAME_RESOURCE1,
+                INVARIANT_RESOURCE1_UUID, "example/sdc/blueprint-dcae/tca-guilin.yaml", "tca-guilin.yaml",
+                INVARIANT_SERVICE_UUID);
+        listResources.add(blueprintArtifact.getResourceAttached());
+        blueprintMap.put(blueprintArtifact.getBlueprintArtifactName(), blueprintArtifact);
+
+        // Build fake csarhandler
+        Mockito.when(csarHandler.getSdcNotification()).thenReturn(notificationData);
+        // Build fake csar Helper
+        ISdcCsarHelper csarHelper = Mockito.mock(ISdcCsarHelper.class);
+        Metadata data = Mockito.mock(Metadata.class);
+        Mockito.when(data.getValue("name")).thenReturn(generatedName);
+        Mockito.when(notificationData.getServiceName()).thenReturn(generatedName);
+        Mockito.when(csarHelper.getServiceMetadata()).thenReturn(data);
+
+        // Create helper based on real csar to test policy yaml and global properties
+        // set
+        SdcToscaParserFactory factory = SdcToscaParserFactory.getInstance();
+        String path = Thread.currentThread().getContextClassLoader().getResource(csarFileName).getFile();
+        ISdcCsarHelper sdcHelper = factory.getSdcCsarHelper(path);
+        Mockito.when(csarHandler.getSdcCsarHelper()).thenReturn(sdcHelper);
+
+        // Mockito.when(csarHandler.getSdcCsarHelper()).thenReturn(csarHelper);
+        Mockito.when(csarHandler.getPolicyModelYaml())
+                .thenReturn(Optional.ofNullable(ResourceFileUtils.getResourceAsString("tosca/tosca_example.yaml")));
+        return csarHandler;
+    }
+
     private CsarHandler buildFakeCsarHandler(String generatedName, String csarFileName) throws IOException,
             SdcToscaParserException {
         // Create fake notification
@@ -203,14 +245,25 @@ public class CsarInstallerItCase {
         assertThat(csarInstaller.isCsarAlreadyDeployed(csarHandler)).isTrue();
     }
 
+    @Test(expected = SdcArtifactInstallerException.class)
+    @Transactional
+    public void testInstallTheBadCsarTca()
+            throws IOException, SdcToscaParserException, InterruptedException, BlueprintParserException,
+            SdcArtifactInstallerException {
+        // This test validates that the blueprint is well rejected because the blueprint contains a link
+        // to a policy that does not exist on the policy engine emulator.
+        String generatedName = RandomStringUtils.randomAlphanumeric(5);
+        csarInstaller.installTheCsar(buildBadFakeCsarHandler(generatedName, CSAR_ARTIFACT_NAME_NO_CDS));
+    }
+
     @Test
     @Transactional
     @Commit
     public void testInstallTheCsarTca() throws SdcArtifactInstallerException, SdcToscaParserException,
             CsarHandlerException, IOException, JSONException, InterruptedException, BlueprintParserException {
         String generatedName = RandomStringUtils.randomAlphanumeric(5);
-        CsarHandler csar = buildFakeCsarHandler(generatedName, CSAR_ARTIFACT_NAME_CDS);
-        csarInstaller.installTheCsar(csar);
+        csarInstaller.installTheCsar(buildFakeCsarHandler(generatedName, CSAR_ARTIFACT_NAME_CDS));
+
         assertThat(serviceRepository.existsById("63cac700-ab9a-4115-a74f-7eac85e3fce0")).isTrue();
         // We should have CDS info
         assertThat(serviceRepository.findById("63cac700-ab9a-4115-a74f-7eac85e3fce0").get().getResourceByType("VF")

diff --git a/src/test/resources/example/sdc/blueprint-dcae/tca-guilin.yaml b/src/test/resources/example/sdc/blueprint-dcae/tca-guilin.yaml
new file mode 100644 (file)
index 0000000..e7d967a
--- /dev/null
+++ b/src/test/resources/example/sdc/blueprint-dcae/tca-guilin.yaml
@@ -0,0 +1,141 @@
+# ============LICENSE_START====================================================
+# =============================================================================
+# Copyright (C) 2019-2020 AT&T Intellectual Property. All rights reserved.
+# =============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END======================================================
+#k8s-tca-gen2-v3.yaml
+
+tosca_definitions_version: cloudify_dsl_1_3
+imports:
+  - https://www.getcloudify.org/spec/cloudify/4.5.5/types.yaml
+  - plugin:k8splugin?version=3.4.2
+  - plugin:clamppolicyplugin?version=1.1.0
+inputs:
+  service_name:
+    type: string
+    default: 'dcae-tcagen2'
+  log_directory:
+    type: string
+    default: "/opt/logs/dcae-analytics-tca"
+  replicas:
+    type: integer
+    description: number of instances
+    default: 1
+  spring.data.mongodb.uri:
+    type: string
+    default: "mongodb://dcae-mongohost/dcae-tcagen2"
+  tag_version:
+    type: string
+    default: "nexus3.onap.org:10001/onap/org.onap.dcaegen2.analytics.tca-gen2.dcae-analytics-tca-web:1.2.1"
+  tca.aai.password:
+    type: string
+    default: "DCAE"
+  tca.aai.url:
+    type: string
+    default: "http://aai.onap.svc.cluster.local"
+  tca.aai.username:
+    type: string
+    default: "DCAE"
+  tca_handle_in_subscribe_url:
+    type: string
+    default: "http://message-router.onap.svc.cluster.local:3904/events/unauthenticated.VES_MEASUREMENT_OUTPUT"
+  tca_handle_out_publish_url:
+    type: string
+    default: "http://message-router.onap.svc.cluster.local:3904/events/unauthenticated.DCAE_CL_OUTPUT"
+  tca_consumer_group:
+    type: string
+    default: "cg1"
+  policy_model_id:
+    type: string
+    default: "onap.policies.monitoring.tcagen2"
+  policy_id:
+    type: string
+    default: "onap.restart.tca"
+node_templates:
+  docker.tca:
+    type: dcae.nodes.ContainerizedServiceComponent
+    relationships:
+      - target: tcagen2_policy
+        type: cloudify.relationships.depends_on
+    interfaces:
+      cloudify.interfaces.lifecycle:
+        start:
+          inputs:
+            ports:
+              - concat: ["9091:", "0"]
+    properties:
+      application_config:
+        service_calls: []
+        streams_publishes:
+          tca_handle_out:
+            dmaap_info:
+              topic_url:
+                get_input: tca_handle_out_publish_url
+            type: message_router
+        streams_subscribes:
+          tca_handle_in:
+            dmaap_info:
+              topic_url:
+                get_input: tca_handle_in_subscribe_url
+            type: message_router
+        spring.data.mongodb.uri:
+          get_input: spring.data.mongodb.uri
+        streams_subscribes.tca_handle_in.consumer_group:
+          get_input: tca_consumer_group
+        streams_subscribes.tca_handle_in.consumer_ids[0]: c0
+        streams_subscribes.tca_handle_in.consumer_ids[1]: c1
+        streams_subscribes.tca_handle_in.message_limit: 50000
+        streams_subscribes.tca_handle_in.polling.auto_adjusting.max: 60000
+        streams_subscribes.tca_handle_in.polling.auto_adjusting.min: 30000
+        streams_subscribes.tca_handle_in.polling.auto_adjusting.step_down: 30000
+        streams_subscribes.tca_handle_in.polling.auto_adjusting.step_up: 10000
+        streams_subscribes.tca_handle_in.polling.fixed_rate: 0
+        streams_subscribes.tca_handle_in.timeout: -1
+        tca.aai.enable_enrichment: true
+        tca.aai.generic_vnf_path: aai/v11/network/generic-vnfs/generic-vnf
+        tca.aai.node_query_path: aai/v11/search/nodes-query
+        tca.aai.password:
+          get_input: tca.aai.password
+        tca.aai.url:
+          get_input: tca.aai.url
+        tca.aai.username:
+          get_input: tca.aai.username
+        tca.policy: '{"domain":"measurementsForVfScaling","metricsPerEventName":[{"eventName":"vFirewallBroadcastPackets","controlLoopSchemaType":"VM","policyScope":"DCAE","policyName":"DCAE.Config_tca-hi-lo","policyVersion":"v0.0.1","thresholds":[{"closedLoopControlName":"ControlLoop-vFirewall-d0a1dfc6-94f5-4fd4-a5b5-4630b438850a","version":"1.0.2","fieldPath":"$.event.measurementsForVfScalingFields.vNicPerformanceArray[*].receivedTotalPacketsDelta","thresholdValue":300,"direction":"LESS_OR_EQUAL","severity":"MAJOR","closedLoopEventStatus":"ONSET"},{"closedLoopControlName":"ControlLoop-vFirewall-d0a1dfc6-94f5-4fd4-a5b5-4630b438850a","version":"1.0.2","fieldPath":"$.event.measurementsForVfScalingFields.vNicPerformanceArray[*].receivedTotalPacketsDelta","thresholdValue":700,"direction":"GREATER_OR_EQUAL","severity":"CRITICAL","closedLoopEventStatus":"ONSET"}]},{"eventName":"vLoadBalancer","controlLoopSchemaType":"VM","policyScope":"DCAE","policyName":"DCAE.Config_tca-hi-lo","policyVersion":"v0.0.1","thresholds":[{"closedLoopControlName":"ControlLoop-vDNS-6f37f56d-a87d-4b85-b6a9-cc953cf779b3","version":"1.0.2","fieldPath":"$.event.measurementsForVfScalingFields.vNicPerformanceArray[*].receivedTotalPacketsDelta","thresholdValue":300,"direction":"GREATER_OR_EQUAL","severity":"CRITICAL","closedLoopEventStatus":"ONSET"}]},{"eventName":"Measurement_vGMUX","controlLoopSchemaType":"VNF","policyScope":"DCAE","policyName":"DCAE.Config_tca-hi-lo","policyVersion":"v0.0.1","thresholds":[{"closedLoopControlName":"ControlLoop-vCPE-48f0c2c3-a172-4192-9ae3-052274181b6e","version":"1.0.2","fieldPath":"$.event.measurementsForVfScalingFields.additionalMeasurements[*].arrayOfFields[0].value","thresholdValue":0,"direction":"EQUAL","severity":"MAJOR","closedLoopEventStatus":"ABATED"},{"closedLoopControlName":"ControlLoop-vCPE-48f0c2c3-a172-4192-9ae3-052274181b6e","version":"1.0.2","fieldPath":"$.event.measurementsForVfScalingFields.additionalMeasurements[*].arrayOfFields[0].value","thresholdValue":0,"direction":"GREATER","severity":"CRITICAL","closedLoopEventStatus":"ONSET"}]}]}'
+        tca.processing_batch_size: 10000
+        tca.enable_abatement: true
+        tca.enable_ecomp_logging: true
+      docker_config:
+        healthcheck:
+          endpoint: /actuator/health
+          interval: 30s
+          timeout: 10s
+          type: http
+      image:
+        get_input: tag_version
+      log_info:
+        log_directory:
+          get_input: log_directory
+      tls_info:
+        use_tls: true
+        cert_directory: '/etc/tca-gen2/ssl'
+      replicas:
+        get_input: replicas
+      service_component_type: { get_input: service_name }
+  tcagen2_policy:
+    type: clamp.nodes.policy
+    properties:
+      policy_id:
+        get_input: policy_id
+      policy_model_id:
+        get_input: policy_model_id