update sdc logic for new types 93/51193/1
authorMichael Lando <ml636r@att.com>
Mon, 11 Jun 2018 10:26:35 +0000 (13:26 +0300)
committerMichael Lando <ml636r@att.com>
Tue, 12 Jun 2018 09:38:22 +0000 (12:38 +0300)
update sdc conformance level and add logic to package onap normatives.

Change-Id: I321eeb0cccf662346eb36cb42420fd8fca72ae32
Issue-ID: SDC-1380
Signed-off-by: Michael Lando <ml636r@att.com>
asdctool/pom.xml
asdctool/sdc-cassandra-init/chef-repo/cookbooks/cassandra-actions/recipes/04-importComformance.rb
asdctool/sdc-cassandra-init/chef-repo/cookbooks/cassandra-actions/templates/default/configuration.yaml.erb
asdctool/src/main/java/org/openecomp/sdc/asdctool/main/SdcSchemaFileImport.java
asdctool/src/test/java/org/openecomp/sdc/asdctool/main/SdcSchemaFileImportTest.java
catalog-be/sdc-backend/chef-repo/cookbooks/sdc-catalog-be/templates/default/BE-configuration.yaml.erb
catalog-be/src/main/resources/config/configuration.yaml
catalog-be/src/main/resources/import/tosca/data-types/dataTypes.yml
catalog-be/src/main/resources/import/tosca/data-types/dataTypes.zip
docs/images/ONAP Proposal_DataFiles_and_3gppPMs_June8 (Read-Only).pdf [new file with mode: 0644]

index 4fd16cc..ce64b7f 100644 (file)
        <build>
                <finalName>${project.artifactId}-${project.version}-jar-with-dependencies</finalName>
                <plugins>
+                       <plugin>
+                               <artifactId>maven-clean-plugin</artifactId>
+                               <version>3.0.0</version>
+                               <executions>
+                                       <execution>
+                                               <id>clean-static-files</id>
+                                               <phase>clean</phase>
+                                               <goals>
+                                                       <goal>clean</goal>
+                                               </goals>
+                                               <configuration>
+                                                       <filesets>
+                                                               <!-- tosca files -->
+                                                               <fileset>
+                                                                       <directory>${project.parent.basedir}/asdctool/tosca</directory>
+                                                                       <followSymlinks>false</followSymlinks>
+                                                               </fileset>
+                                                               <fileset>
+                                                                       <directory>${project.basedir}/sdc-cassandra-init/chef-repo/cookbooks/cassandra-actions/attributes/</directory>
+                                                                       <followSymlinks>false</followSymlinks>
+                                                                       <includes>
+                                                                               <include>**/default.rb</include>
+                                                                       </includes>
+                                                               </fileset>
+                                                               <fileset>
+                                                                       <directory>${project.basedir}/sdc-cassandra-init/chef-repo/cookbooks/cassandra-actions/files/default/tools</directory>
+                                                                       <followSymlinks>false</followSymlinks>
+                                                               </fileset>
+                                                               <fileset>
+                                                                       <directory>${project.basedir}/sdc-cassandra-init/chef-repo/cookbooks/cassandra-actions/files/default</directory>
+                                                                       <followSymlinks>false</followSymlinks>
+                                                                       <includes>
+                                                                               <include>**/sdctool.tar</include>
+                                                                       </includes>
+                                                               </fileset>
+                                                       </filesets>
+                                               </configuration>
+                                       </execution>
+                               </executions>
+                       </plugin>
 
                        <plugin>
                                <groupId>org.apache.maven.plugins</groupId>
                                        </execution>
                                </executions>
                        </plugin>
-
-                       <plugin>
-                               <artifactId>maven-clean-plugin</artifactId>
-                               <version>3.0.0</version>
-                               <executions>
-                                       <execution>
-                                               <id>clean-static-files</id>
-                                               <phase>clean</phase>
-                                               <goals>
-                                                       <goal>clean</goal>
-                                               </goals>
-                                               <configuration>
-                                                       <filesets>
-                                                               <!-- tosca files -->
-                                                               <fileset>
-                                                                       <directory>${project.parent.basedir}/asdctool/tosca</directory>
-                                                                       <followSymlinks>false</followSymlinks>
-                                                               </fileset>
-                                                               <fileset>
-                                                                       <directory>${project.basedir}/sdc-cassandra-init/chef-repo/cookbooks/cassandra-actions/attributes/</directory>
-                                                                       <followSymlinks>false</followSymlinks>
-                                                                       <includes>
-                                                                               <include>**/default.rb</include>
-                                                                       </includes>
-                                                               </fileset>
-                                                               <fileset>
-                                                                       <directory>${project.basedir}/sdc-cassandra-init/chef-repo/cookbooks/cassandra-actions/files/default/tools</directory>
-                                                                       <followSymlinks>false</followSymlinks>
-                                                               </fileset>
-                                                               <fileset>
-                                                                       <directory>${project.basedir}/sdc-cassandra-init/chef-repo/cookbooks/cassandra-actions/files/default</directory>
-                                                                       <followSymlinks>false</followSymlinks>
-                                                                       <includes>
-                                                                               <include>**/sdctool.tar</include>
-                                                                       </includes>
-                                                               </fileset>
-                                                       </filesets>
-                                               </configuration>
-                                       </execution>
-                               </executions>
-                       </plugin>
                        <plugin>
                                <groupId>ru.yaal.maven</groupId>
                                <artifactId>write-text-files-maven-plugin</artifactId>
                                <artifactId>maven-resources-plugin</artifactId>
                                <version>3.0.2</version>
                                <executions>
-
                                        <execution>
                                                <id>copy-tosca-folder</id>
                                                <!-- here the phase you need -->
                                                        </resources>
                                                </configuration>
                                        </execution>
-
                                </executions>
                        </plugin>
                </plugins>
        </build>
        <profiles>
-               <profile>
-                       <id>docker-staging</id>
-                       <properties>
-                               <docker.tag>${project.version}-STAGING-${maven.build.timestamp}</docker.tag>
-                               <docker.latest.tag>${project.version}-STAGING-latest</docker.latest.tag>
-                       </properties>
-               </profile>
                <profile>
                        <id>docker</id>
                        <activation>
                                                <artifactId>maven-resources-plugin</artifactId>
                                                <version>3.0.2</version>
                                                <executions>
-                                                       <execution>
-                                                               <id>copy-dox-files</id>
-                                                               <phase>verify</phase>
-                                                               <goals>
-                                                                       <goal>copy-resources</goal>
-                                                               </goals>
-                                                               <configuration>
-                                                                       <outputDirectory>${basedir}/sdc-cassandra-init/chef-repo/cookbooks/cassandra-actions/files/default</outputDirectory>
-                                                                       <resources>
-                                                                               <resource>
-                                                                                       <directory>${project.parent.basedir}/openecomp-be</directory>
-                                                                                       <includes>
-                                                                                               <include>tools/build/**</include>
-                                                                                       </includes>
-                                                                               </resource>
-                                                                               <resource>
-                                                                                       <directory>${project.parent.basedir}/openecomp-be</directory>
-                                                                                       <includes>
-                                                                                               <include>tools/install/**</include>
-                                                                                       </includes>
-                                                                               </resource>
-                                                                       </resources>
-                                                               </configuration>
-                                                       </execution>
                                                        <execution>
                                                                <id>copy-sdctool</id>
                                                                <phase>verify</phase>
                                                <groupId>io.fabric8</groupId>
                                                <artifactId>docker-maven-plugin</artifactId>
                                                <version>${fabric8.version}</version>
-
                                                <configuration>
                                                        <verbose>true</verbose>
                                                        <apiVersion>1.23</apiVersion>
index 3b293e8..6841175 100644 (file)
@@ -14,7 +14,7 @@ bash "import-Comformance" do
 
     cd /tmp/sdctool/scripts
     /bin/chmod +x sdcSchemaFileImport.sh
-    echo "execute /tmp/sdctool/scripts/sdcSchemaFileImport.sh ${tosca_dir} #{cl_release} ${cl_version} ${conf_dir} "
+    echo "execute /tmp/sdctool/scripts/sdcSchemaFileImport.sh ${tosca_dir} #{cl_release} ${cl_version} ${conf_dir} onap"
     ./sdcSchemaFileImport.sh ${tosca_dir} #{cl_release} ${cl_version} ${conf_dir}
   EOH
 end
\ No newline at end of file
index 75a5d75..d6e4c11 100644 (file)
@@ -16,13 +16,17 @@ beHttpPort: <%= @catalog_port %>
 beContext: /sdc/rest/config/get
 
 # catalog backend protocol
+<% if node[:disableHttp] -%>
+beProtocol: https
+<% else %>
 beProtocol: http
+<% end -%>
 
 # catalog backend ssl port
 beSslPort: <%= @ssl_port %>
 version: 1.0
 released: 2012-11-30
-toscaConformanceLevel: 5.0
+toscaConformanceLevel: 10.0
 minToscaConformanceLevel: 3.0
 
 titanCfgFile: <%= @titan_Path %>/titan.properties
@@ -130,6 +134,7 @@ artifactTypes:
    - SNMP_POLL
    - SNMP_TRAP
    - GUIDE
+   - PLAN
 
 licenseTypes:
    - User
@@ -146,23 +151,9 @@ resourceTypes: &allResourceTypes
   - Abstract
   - CVFC
 
-# validForResourceTypes usage
-#     validForResourceTypes:
-#        - VF
-#        - VL
+
 deploymentResourceArtifacts:
-#  heat:
-#      displayName: "Base HEAT Template"
-#      type: HEAT
-#      validForResourceTypes: *allResourceTypes
-#  heatVol:
-#      displayName: "Volume HEAT Template"
-#      type: HEAT_VOL
-#      validForResourceTypes: *allResourceTypes
-#  heatNet:
-#      displayName: "Network HEAT Template"
-#      type: HEAT_NET
-#      validForResourceTypes: *allResourceTypes
+
 
 deploymentResourceInstanceArtifacts:
   heatEnv:
@@ -193,7 +184,7 @@ toscaArtifacts:
 excludeResourceCategory:
   - Generic
 excludeResourceType:
-  - PNF  
+  - PNF
 informationalResourceArtifacts:
   features:
       displayName: Features
@@ -283,7 +274,7 @@ serviceApiArtifacts:
 additionalInformationMaxNumberOfKeys: 50
 
 systemMonitoring:
-    enabled: true
+    enabled: false
     isProxy: false
     probeIntervalInSeconds: 15
 
@@ -302,9 +293,6 @@ serviceDeploymentArtifacts:
     MODEL_QUERY_SPEC:
         acceptedTypes:
             - xml
-    UCPE_LAYER_2_CONFIGURATION:
-        acceptedTypes:
-            - xml
 
 #AAI Artifacts
     AAI_SERVICE_MODEL:
@@ -316,9 +304,17 @@ serviceDeploymentArtifacts:
     AAI_VF_INSTANCE_MODEL:
         acceptedTypes:
             - xml
+    UCPE_LAYER_2_CONFIGURATION:
+        acceptedTypes:
+            - xml
     OTHER:
         acceptedTypes:
 
+#PLAN
+    PLAN:
+        acceptedTypes:
+            - xml
+
 resourceDeploymentArtifacts:
     HEAT:
         acceptedTypes:
@@ -436,6 +432,14 @@ resourceDeploymentArtifacts:
         acceptedTypes:
         validForResourceTypes: *allResourceTypes
 
+#PLAN
+    PLAN:
+        acceptedTypes:
+            - xml
+        validForResourceTypes:
+            - VF
+            - VFC
+
 resourceInstanceDeploymentArtifacts:
     HEAT_ENV:
         acceptedTypes:
@@ -473,6 +477,11 @@ resourceInstanceDeploymentArtifacts:
         acceptedTypes:
         validForResourceTypes: *allResourceTypes
 
+#PLAN
+    PLAN:
+        acceptedTypes:
+            - xml
+
 resourceInformationalArtifacts:
     CHEF:
         acceptedTypes:
@@ -503,7 +512,7 @@ resourceInformationalArtifacts:
         validForResourceTypes: *allResourceTypes
     OTHER:
         acceptedTypes:
-        validForResourceTypes: 
+        validForResourceTypes:
             - VFC
             - CVFC
             - CP
@@ -583,9 +592,9 @@ applicationL1Cache:
         pollIntervalInSec: 60
 
 applicationL2Cache:
-    enabled: true
+    enabled: false
     catalogL1Cache:
-        enabled: true
+        enabled: false
         resourcesSizeInCache: 300
         servicesSizeInCache: 200
         productsSizeInCache: 100
@@ -637,4 +646,62 @@ environmentContext:
        - Important_Non-Revenue
        - Needed_Non-Revenue
        - Useful_Non-Revenue
-       - General_Non-Revenue
\ No newline at end of file
+       - General_Non-Revenue
+
+dmaapConsumerConfiguration:
+    hosts: localhost:3905
+    consumerGroup: sdc
+    consumerId: mama
+    timeoutMs: 15000
+    limit: 1
+    pollingInterval: 2
+    topic: topic
+    latitude: 32.109333
+    longitude: 34.855499
+    version: 1.0
+    serviceName: localhost/events
+    environment: TEST
+    partner: BOT_R
+    routeOffer: MR1
+    protocol: https
+    contenttype: application/json
+    dme2TraceOn: true
+    aftEnvironment: AFTUAT
+    aftDme2ConnectionTimeoutMs: 15000
+    aftDme2RoundtripTimeoutMs: 240000
+    aftDme2ReadTimeoutMs: 50000
+    dme2preferredRouterFilePath: DME2preferredRouter.txt
+    timeLimitForNotificationHandleMs: 120000
+    credential:
+        username: user
+        password:
+
+dmeConfiguration:
+    dme2Search: DME2SEARCH
+    dme2Resolve: DME2RESOLVE
+
+excludedPolicyTypesMapping:
+   # VF:
+    #  - a.b.c
+    #  - c.d.e
+    #CR:
+    #  - x.y.z
+
+excludedGroupTypesMapping:
+    CR:
+       - org.openecomp.groups.VfModule
+       - org.openecomp.groups.heat.HeatStack
+       - tosca.groups.Root
+    VF:
+       - org.openecomp.groups.VfModule
+       - org.openecomp.groups.heat.HeatStack
+       - tosca.groups.Root
+    Service:
+       - org.openecomp.groups.VfModule
+       - org.openecomp.groups.heat.HeatStack
+       - tosca.groups.Root
+
+healthStatusExclude:
+   - DE
+   - DMAPP
+   - DCAE
index 0e545f2..e915d27 100644 (file)
@@ -36,6 +36,7 @@ import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
 import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang3.ArrayUtils;
 import org.openecomp.sdc.asdctool.enums.SchemaZipFileEnum;
 import org.openecomp.sdc.asdctool.impl.EsToCassandraDataMigrationConfig;
 import org.openecomp.sdc.be.config.ConfigurationManager;
@@ -53,16 +54,16 @@ import org.yaml.snakeyaml.Yaml;
 public class SdcSchemaFileImport {
        
        private static final String SEPARATOR = FileSystems.getDefault().getSeparator();
-
-       private static SdcSchemaFilesCassandraDao schemaFilesCassandraDao;
        
        private static final String TOSCA_VERSION = "tosca_simple_yaml_1_1";
                
        private static String importToscaPath;
        
-       private static byte[] buffer = new byte[1024];
+       private static final byte[] buffer = new byte[1024];
        
-       private static String YAML_EXTENSION = ".yml";
+       private static final String YAML_EXTENSION = ".yml";
+
+       private static final String DEPLOYMENT_TYPE_ONAP = "onap";
        
        private static String LICENSE_TXT;
        
@@ -74,7 +75,7 @@ public class SdcSchemaFileImport {
                System.out.println("Starting SdcSchemaFileImport procedure...");
                final String FILE_NAME = "SDC.zip";
                
-               if (args == null || args.length < 4) {
+               if (args == null || !(args.length ==4 || args.length == 5 )) {
                        usageAndExit();
                }
                
@@ -82,6 +83,11 @@ public class SdcSchemaFileImport {
                String sdcReleaseNum = args[1];
                String conformanceLevel = args[2];
                String appConfigDir = args[3];
+               String deploymentType=null;
+               if(args.length==5){
+                       deploymentType=args[4];
+               }
+
                                
                ByteArrayOutputStream baos = new ByteArrayOutputStream();
 
@@ -116,7 +122,7 @@ public class SdcSchemaFileImport {
                        }
                }
                
-               createAndSaveNodeSchemaFile();
+               createAndSaveNodeSchemaFile(deploymentType);
                
        try  {
                //close the ZipOutputStream
@@ -128,10 +134,10 @@ public class SdcSchemaFileImport {
                System.exit(1);
        }
                
-       //Generation flow end - generating SDC from normatives
+       //Generation flow end - generating SDC from narratives
                                
                AnnotationConfigApplicationContext context = initContext(appConfigDir);
-               schemaFilesCassandraDao = (SdcSchemaFilesCassandraDao) context.getBean("sdc-schema-files-cassandra-dao");
+        SdcSchemaFilesCassandraDao schemaFilesCassandraDao = (SdcSchemaFilesCassandraDao) context.getBean("sdc-schema-files-cassandra-dao");
                
                byte[] fileBytes = baos.toByteArray();
 
@@ -166,7 +172,7 @@ public class SdcSchemaFileImport {
                yaml.setName(fileName);
                 
                //Initialize the yaml contents
-               Map<String, Object> data = new LinkedHashMap<String, Object>();
+               Map<String, Object> data = new LinkedHashMap<>();
                                        
                data.put("tosca_definitions_version", TOSCA_VERSION);
                
@@ -199,29 +205,40 @@ public class SdcSchemaFileImport {
                        zos.write(buffer, 0, len);
                }
                //close the InputStream
-               stream.close();
+            file.delete();
+            stream.close();
                zos.closeEntry();
-               file.delete();
+
                
                } catch (IOException e) {
                        System.out.println("Error in file creation : " + fileName + ", " + e.getMessage());
                        System.exit(1);
                }
        }
-       
-       public static void createAndSaveNodeSchemaFile() throws IOException  {
+
+       /**
+        *the method is responsible for creating and storing the sdc normatives in the DB
+        * @param deploymentType if the deployments type is onap the onap narratives will be add to the zip
+        * @throws IOException thrown in case of issues in reding files.
+        */
+       public static void createAndSaveNodeSchemaFile(String deploymentType) throws IOException  {
                
                //Initialize the snake yaml dumper option
                DumperOptions options = new DumperOptions();
                options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK);
                
-               Map<String, Object> nodeTypeList = new LinkedHashMap<String, Object>();
+               Map<String, Object> nodeTypeList = new LinkedHashMap<>();
                
                String[] importFileList = new String[]{"data.yml", "artifacts.yml", "capabilities.yml", "interfaces.yml", "relationships.yml"}; 
                String collectionTitle = "node_types";
                
                //Create node.yaml - collect all types from normative-types and heat-types directories
                String[] nodeTypesMainFolders = new String[]{"normative-types", "heat-types"};
+
+               if(DEPLOYMENT_TYPE_ONAP.equals(deploymentType)){
+            String[] onapNodeTypesMainFolders = new String[]{"nfv-types"};
+            nodeTypesMainFolders=ArrayUtils.addAll(nodeTypesMainFolders,onapNodeTypesMainFolders);
+               }
                
                for (String nodeTypesMainFolder : nodeTypesMainFolders) {
                        Files.walk(Paths.get(importToscaPath + SEPARATOR + nodeTypesMainFolder))
@@ -232,7 +249,7 @@ public class SdcSchemaFileImport {
                                                System.out.println("Processing node type file "+path+"...");
                                                FileInputStream inputStream = new FileInputStream(path);
                                        Yaml yaml = new Yaml();
-                                       Map<String, Object> load = (Map<String, Object>) yaml.load(inputStream);
+                                       Map<String, Object> load = yaml.loadAs(inputStream,Map.class);
                                                Map<String, Object> nodeType = (Map<String, Object>) load.get(collectionTitle);
                                                nodeTypeList.putAll(nodeType);
                                                
@@ -251,13 +268,12 @@ public class SdcSchemaFileImport {
        }
        
        private static void SdcSchemaFileImportUsage()  {
-               System.err.println("Usage: <file dir/filename> <SDC release number> <Schema conformance level> <configuration dir>");
+               System.err.println("Usage: <file dir/filename> <SDC release number> <Schema conformance level> <configuration dir> <deployment type optional>");
        }
        
        private static AnnotationConfigApplicationContext initContext(String appConfigDir) {
                ConfigurationSource configurationSource = new FSConfigurationSource(ExternalConfiguration.getChangeListener(), appConfigDir);
-               ConfigurationManager configurationManager = new ConfigurationManager(configurationSource);
-               AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(EsToCassandraDataMigrationConfig.class);
-               return context;
+               new ConfigurationManager(configurationSource);
+               return  new AnnotationConfigApplicationContext(EsToCassandraDataMigrationConfig.class);
        }
 }
index 2f5daba..791aeb9 100644 (file)
@@ -15,7 +15,14 @@ public class SdcSchemaFileImportTest {
        public void testCreateAndSaveNodeSchemaFile() throws Exception {
 
                // default test
-               SdcSchemaFileImport.createAndSaveNodeSchemaFile();
+               SdcSchemaFileImport.createAndSaveNodeSchemaFile("");
+       }
+
+       @Test(expected=NoSuchFileException.class)
+       public void testCreateAndSaveNodeSchemaFileOnap() throws Exception {
+
+               // default test
+               SdcSchemaFileImport.createAndSaveNodeSchemaFile("onap");
        }
 
        @Test(expected=NullPointerException.class)
index 1789507..4d86fbf 100644 (file)
@@ -25,7 +25,7 @@ beProtocol: http
 beSslPort: <%= @ssl_port %>
 version: 1.1.0
 released: 2012-11-30
-toscaConformanceLevel: 5.0
+toscaConformanceLevel: 10.0
 minToscaConformanceLevel: 3.0
 
 titanCfgFile: /var/lib/jetty/config/catalog-be/titan.properties
index aefdace..991ed92 100644 (file)
@@ -23,7 +23,7 @@ beSslPort: 8443
 
 version: 1.1.0
 released: 2012-11-30
-toscaConformanceLevel: 7.0
+toscaConformanceLevel: 10.0
 minToscaConformanceLevel: 3.0
 
 titanCfgFile: /home/vagrant/catalog-be/config/catalog-be/titan.properties
index bbac8b7..697b333 100644 (file)
@@ -1371,6 +1371,8 @@ tosca.datatypes.nfv.VirtualNetworkInterfaceRequirements:
     support_mandatory:
       type: boolean
       required: true
+    requirement:
+      type: string      # not specified
     network_interface_requirements:
       type: map
       entry_schema:
@@ -1394,6 +1396,9 @@ tosca.datatypes.nfv.L2AddressData:
     mac_address_assignment: 
       type: boolean
       required: true
+    address:
+      type: string
+      required: false
 tosca.datatypes.nfv.L3AddressData:
   derived_from: tosca.datatypes.Root
   properties: 
index bd8b337..1e1fa1e 100644 (file)
Binary files a/catalog-be/src/main/resources/import/tosca/data-types/dataTypes.zip and b/catalog-be/src/main/resources/import/tosca/data-types/dataTypes.zip differ
diff --git a/docs/images/ONAP Proposal_DataFiles_and_3gppPMs_June8 (Read-Only).pdf b/docs/images/ONAP Proposal_DataFiles_and_3gppPMs_June8 (Read-Only).pdf
new file mode 100644 (file)
index 0000000..dfd7949
Binary files /dev/null and b/docs/images/ONAP Proposal_DataFiles_and_3gppPMs_June8 (Read-Only).pdf differ