Ground work for hpa validation

Added the declarative schema for hpa validation. This is the ground
layering work for hpa validation.

Change-Id: I2ff30489e191ecf187711e1025f6e3faee9946c3
Issue-ID: VNFSDK-194
Signed-off-by: Lianhao Lu <lianhao.lu@intel.com>

diff --git a/MANIFEST.in b/MANIFEST.in
index ff36802..5624cf9 100644 (file)
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,3 +1,5 @@
 include COPYRIGHT
 include LICENSE.txt
 include requirements.txt
+include vnfsdk_pkgtools/validator/*.yaml
+prune tests/

diff --git a/requirements.txt b/requirements.txt
index 45562d5..e8df081 100644 (file)
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-ruamel.yaml<0.16
+ruamel.yaml<0.15
 requests>=2.3.0
 #apache-ariatosca==0.1.1
 stevedore >= 1.9.0

diff --git a/setup.py b/setup.py
index b9db75a..3e0f38f 100644 (file)
--- a/setup.py
+++ b/setup.py
@@ -80,6 +80,7 @@ setup(
         'Topic :: System :: Systems Administration'],
 
     packages=find_packages(exclude=['tests*']),
+    package_data={'vnfsdk_pkgtools.validator': ['*.yaml']},
 
     entry_points={
         'console_scripts': [

diff --git a/tests/validator/test_validate_utils.py b/tests/validator/test_validate_utils.py
new file mode 100644 (file)
index 0000000..2c36a3b
--- /dev/null
+++ b/tests/validator/test_validate_utils.py
@@ -0,0 +1,42 @@
+# Copyright (c) 2018 Intel Corp. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+import os
+
+from ruamel import yaml
+import pytest
+
+from vnfsdk_pkgtools.validator import utils
+
+CONTENT = "---\n1"
+
+BAD_CONTENT = "---\na: b\n- d"
+
+def test_load_definition(tmpdir):
+    p = tmpdir.join("test_definition.yaml")
+    p.write(CONTENT)
+    assert 1 == utils.load_definitions(str(p))
+
+
+def test_load_bad_definition(tmpdir):
+    p = tmpdir.join("test_definition.yaml")
+    p.write(BAD_CONTENT)
+    with pytest.raises(yaml.YAMLError):
+        utils.load_definitions(str(p))
+
+
+def test_load_defualt_definition(tmpdir):
+    p = tmpdir.join("non_exist")
+    assert 1 == utils.load_definitions(str(p),defaults=1)

diff --git a/vnfsdk_pkgtools/validator/__init__.py b/vnfsdk_pkgtools/validator/__init__.py
index f6d9073..730acf6 100644 (file)
--- a/vnfsdk_pkgtools/validator/__init__.py
+++ b/vnfsdk_pkgtools/validator/__init__.py
@@ -38,7 +38,7 @@ class ValidatorBase(object):
     """Base class for validators."""
 
     def __init__(self):
-        pass
+        self.tosca = None
 
 
     @abc.abstractmethod

diff --git a/vnfsdk_pkgtools/validator/hpa.yaml b/vnfsdk_pkgtools/validator/hpa.yaml
new file mode 100644 (file)
index 0000000..bc551c6
--- /dev/null
+++ b/vnfsdk_pkgtools/validator/hpa.yaml
@@ -0,0 +1,267 @@
+---
+metadata:
+  version: "1.0"
+schemas:
+  # schema name
+  vduCpuRequirements:
+      # hpa key name
+      cpuModelSpecificationBinding:
+        # json encoded key name: reg expression for the valid value
+        schema-version: &any r'.*'
+        schema-location: *any
+        platform-id: &generic r'generic'
+        mandatory: &bool r'true|false|TRUE|FALSE|True|False'
+        configuration-value: r'strictBinding|equalOrBetterBinding'
+      instructionSetRequirements:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: *any
+      simultaneousMultiThreading:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: r'enabled|disabled'
+      hypervisorConfiguration:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: *any
+      computeRas:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: r'pciDetectedAndCorrectedErrors|pciDetectedAndUncorrectedErrors'
+      cpuModel:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: *any
+      directIoAccessToCache:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: *any
+      accelerator:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: *any
+      measuredLaunchEnvironment:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: *any
+      secureEnclave:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: *any
+      numVirtualCpu:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: r'\d+'
+      virtualCpuClock:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: r'\d+\w*(Hz|kHz|MHz|GHz)'
+      logicalCpuPinningPolicy:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: r'dedicated|shared'
+      logicalCpuThreadPinningPolicy:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: r'require|isolate|prefer'
+  vduMemRequirements:
+      memoryPageSize:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: r'\d+\w*(B|kB|KB|KiB|MB|MiB|GB|GiB|TB|TiB)'
+      numberOfPages:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: r'\d+'
+      memoryAllocationPolicy:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: r'strictLocalAffinity|preferredLocalAffinity'
+      memoryType:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: *any
+      memorySpeed:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: *any
+      memoryRas:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: *any
+      memoryBandwidth:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: r'\d+'
+      processorCacheAllocationType:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: *any
+      processorCacheAllocationSize:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: *any
+  vduStorageRequirements:
+      storageIops:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: r'\d+'
+      storageResilencyMechanism:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: r'erasure|tripleReplication'
+      processorCacheAllocationSize:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: *any
+      processorCacheAllocationSize:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: *any
+  logicalNodeComputeMemoryRequirements:
+      numberCpu:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: r'\d+'
+      localNumaMemorySize:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: r'\d+\w*(B|kB|KB|KiB|MB|MiB|GB|GiB|TB|TiB)'
+  logicalNodeIoRequirements:
+      pciVendorId:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: *any
+      pciDeviceId:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: *any
+      pciNumDevices:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: r'\d+'
+      pciAddress:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: *any
+      pciDeviceLocalToNumaNode:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: r'required|notRequired'
+  networkInterfaceRequirements:
+      nicFeature:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: *any
+      dataProcessingAccelerationLibray:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: r'DPDK'
+      dataProcessingAccelerationLibrayVersion:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: *any
+      interfaceType:
+        schema-version: *any
+        schema-location: *any
+        platform-id: *generic
+        mandatory: *bool
+        configuration-value: r'virtio|PCI-Passthrough|SR-IOV|E1000|RTL8139|PCNET'
+mappings:
+# mapping between property value of a tosca node type and the valid hpa schema
+#   type: tosca node type
+#   key: prop1##prop2##...##propN
+#          Property hierachy within that node type. Prefix of '(capability:)'
+#          in propI means the value is from capability instead of property.
+#          Suffix '[] in propI means the value is a list of propI+1.
+#   schema: schema defined in the above schemas section
+  - type: tosca.nodes.nfv.Vdu.Compute
+    key: capability:virtual_compute##logical_node##logical_node_requirements
+    schema: logicalNodeComputeMemoryRequirements
+  - type: tosca.nodes.nfv.Vdu.Compute
+    key: capability:virtual_compute##virtual_cpu##vdu_cpu_requirements
+    schema: vduCpuRequirements
+  - type: tosca.nodes.nfv.Vdu.Compute
+    key: capability:virtual_compute##virtual_memory##vdu_memory_requirements
+    schema: vduCpuRequirements
+  - type: tosca.nodes.nfv.Vdu.VirtualStorage
+    key: vdu_storage_requirements
+    schema: vduStorageRequirements
+  - type: tosca.nodes.nfv.VduCp
+    key: virtual_network_interface_requirements[]##network_interface_requirements##logical_node_requirements
+    schema: networkInterfaceRequirements
+  - type: tosca.nodes.nfv.VduCp
+    key: virtual_network_interface_requirements[]##nic_io_requirements##logical_node_requirements
+    schema: logicalNodeIoRequirements

diff --git a/vnfsdk_pkgtools/validator/toscaparser_validator.py b/vnfsdk_pkgtools/validator/toscaparser_validator.py
index bcb3e9e..d1aad30 100644 (file)
--- a/vnfsdk_pkgtools/validator/toscaparser_validator.py
+++ b/vnfsdk_pkgtools/validator/toscaparser_validator.py
@@ -15,18 +15,44 @@
 
 import logging
 import os
+import pkg_resources
 
 from toscaparser.common.exception import ValidationError
 from toscaparser.tosca_template import ToscaTemplate
 
 from vnfsdk_pkgtools import validator
+from vnfsdk_pkgtools.validator import utils
 
 LOG = logging.getLogger(__name__)
 
 
+class HpaSchemaDefError(ValueError):
+    pass
+
+
 class ToscaparserValidator(validator.ValidatorBase):
     def __init__(self):
         super(ToscaparserValidator, self).__init__()
+        self._load_hpa_definition()
+
+    def _load_hpa_definition(self):
+        conf = pkg_resources.resource_filename(__name__, "hpa.yaml")
+        defs = utils.load_definitions(conf, {})
+        self.hpa_schema_version = defs.get('metadata', {}).get('version')
+        self.hpa_schemas = defs.get('schemas', {})
+        self.hpa_mappings = defs.get('mappings', [])
+        #validate schema defined in hpa.yaml is correct
+        if not self.hpa_schema_version:
+            msg = "No defined version in metadata"
+            raise HpaSchemaDefError(msg)
+        for mapping in self.hpa_mappings:
+            schema = mapping.get('schema')
+            if schema not in self.hpa_schemas:
+                msg = "schema %s not found in hpa schema definitions" % schema
+                raise HpaSchemaDefError(msg)
+            if not mapping.get('type') or not mapping.get('key'):
+                msg = "type or key not defined in hpa mappings"
+                raise HpaSchemaDefError(msg)
 
     def validate(self, reader):
         entry_path = os.path.join(reader.destination,
@@ -40,3 +66,5 @@ class ToscaparserValidator(validator.ValidatorBase):
         except ValidationError as e:
             LOG.error(e.message)
             raise e
+
+        print self.tosca

diff --git a/vnfsdk_pkgtools/validator/utils.py b/vnfsdk_pkgtools/validator/utils.py
new file mode 100644 (file)
index 0000000..b26a290
--- /dev/null
+++ b/vnfsdk_pkgtools/validator/utils.py
@@ -0,0 +1,49 @@
+# Copyright (c) 2018 Intel Corp Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+import logging
+import os
+
+from ruamel import yaml
+
+LOG = logging.getLogger(__name__)
+
+
+def load_definitions(config_file, defaults=None):
+    if not os.path.exists(config_file):
+        LOG.info("No definitions configuration file %s found! "
+                 "Using default config.", config_file)
+        definition_cfg = defaults
+    else:
+        LOG.debug("Loading definitions configuration file: %s", config_file)
+        with open(config_file) as cf:
+            config = cf.read()
+        try:
+            definition_cfg = yaml.safe_load(config)
+        except yaml.YAMLError as err:
+            if hasattr(err, 'problem_mark'):
+                mark = err.problem_mark
+                errmsg = ("Invalid YAML syntax in Definitions file %(file)s "
+                           "at line: %(line)s, column: %(column)s." %
+                           dict(file=config_file,
+                                line=mark.line + 1,
+                                column=mark.column + 1))
+            else:
+                errmsg = ("YAML error reading Definitions file %s" % config_file)
+            LOG.error(errmsg)
+            raise
+
+    LOG.debug("Definitions: %s", definition_cfg)
+    return definition_cfg