From: Lianhao Lu <lianhao.lu@intel.com>
Date: Fri, 10 Aug 2018 09:04:14 +0000 (+0800)
Subject: Added the logic to validate HPA
X-Git-Tag: 3.0.0-ONAP~14
X-Git-Url: https://gerrit.onap.org/r/gitweb?a=commitdiff_plain;h=9f629ff62c270fd3771c381f78aa36f473e65bb5;p=vnfsdk%2Fpkgtools.git

Added the logic to validate HPA

Added the code logic to validate HPA for opnfv tosca parser. Also
corrected some hpa validation schema errors introduced by the previous
patch.

Change-Id: Icd61d34d7915aa965ec32adfc3c0f1a117dd6f3e
Issue-ID: VNFSDK-194
Signed-off-by: Lianhao Lu <lianhao.lu@intel.com>
---

diff --git a/tests/resources/hpa.csar b/tests/resources/hpa.csar
new file mode 100644
index 0000000..a9558ae
Binary files /dev/null and b/tests/resources/hpa.csar differ
diff --git a/tests/resources/hpa_bad.csar b/tests/resources/hpa_bad.csar
new file mode 100644
index 0000000..4431d61
Binary files /dev/null and b/tests/resources/hpa_bad.csar differ
diff --git a/tests/validator/test_toscaparser_validator.py b/tests/validator/test_toscaparser_validator.py
index c35d1ed..3348d60 100644
--- a/tests/validator/test_toscaparser_validator.py
+++ b/tests/validator/test_toscaparser_validator.py
@@ -15,13 +15,29 @@
 
 import os
 
+import pytest
+
 from vnfsdk_pkgtools.packager import csar
 from vnfsdk_pkgtools.validator import toscaparser_validator
 
 CSAR_PATH = 'tests/resources/test_import.csar'
+HPA_PATH = 'tests/resources/hpa.csar'
+BAD_HPA_PATH = 'tests/resources/hpa_bad.csar'
 
 def test_validate(tmpdir):
     reader = csar._CSARReader(CSAR_PATH, str(tmpdir.mkdir('validate')))
     validator = toscaparser_validator.ToscaparserValidator()
     validator.validate(reader)
     assert hasattr(validator, 'tosca')
+
+def test_validate_hpa(tmpdir):
+    reader = csar._CSARReader(HPA_PATH, str(tmpdir.mkdir('validate')))
+    validator = toscaparser_validator.ToscaparserValidator()
+    validator.validate(reader)
+    assert hasattr(validator, 'tosca')
+
+def test_validate_hpa_bad(tmpdir):
+    reader = csar._CSARReader(BAD_HPA_PATH, str(tmpdir.mkdir('validate')))
+    validator = toscaparser_validator.ToscaparserValidator()
+    with pytest.raises(toscaparser_validator.HpaValueError):
+        validator.validate(reader)
diff --git a/vnfsdk_pkgtools/validator/hpa.yaml b/vnfsdk_pkgtools/validator/hpa.yaml
index bc551c6..98ac42b 100644
--- a/vnfsdk_pkgtools/validator/hpa.yaml
+++ b/vnfsdk_pkgtools/validator/hpa.yaml
@@ -7,11 +7,11 @@ schemas:
       # hpa key name
       cpuModelSpecificationBinding:
         # json encoded key name: reg expression for the valid value
-        schema-version: &any r'.*'
+        schema-version: &any '.*'
         schema-location: *any
-        platform-id: &generic r'generic'
-        mandatory: &bool r'true|false|TRUE|FALSE|True|False'
-        configuration-value: r'strictBinding|equalOrBetterBinding'
+        platform-id: &generic 'generic'
+        mandatory: &bool 'true|false|TRUE|FALSE|True|False'
+        configuration-value: 'strictBinding|equalOrBetterBinding'
       instructionSetRequirements:
         schema-version: *any
         schema-location: *any
@@ -23,7 +23,7 @@ schemas:
         schema-location: *any
         platform-id: *generic
         mandatory: *bool
-        configuration-value: r'enabled|disabled'
+        configuration-value: 'enabled|disabled'
       hypervisorConfiguration:
         schema-version: *any
         schema-location: *any
@@ -35,7 +35,7 @@ schemas:
         schema-location: *any
         platform-id: *generic
         mandatory: *bool
-        configuration-value: r'pciDetectedAndCorrectedErrors|pciDetectedAndUncorrectedErrors'
+        configuration-value: 'pciDetectedAndCorrectedErrors|pciDetectedAndUncorrectedErrors'
       cpuModel:
         schema-version: *any
         schema-location: *any
@@ -71,44 +71,44 @@ schemas:
         schema-location: *any
         platform-id: *generic
         mandatory: *bool
-        configuration-value: r'\d+'
+        configuration-value: '\d+'
       virtualCpuClock:
         schema-version: *any
         schema-location: *any
         platform-id: *generic
         mandatory: *bool
-        configuration-value: r'\d+\w*(Hz|kHz|MHz|GHz)'
+        configuration-value: '\d+\s*(Hz|kHz|MHz|GHz)'
       logicalCpuPinningPolicy:
         schema-version: *any
         schema-location: *any
         platform-id: *generic
         mandatory: *bool
-        configuration-value: r'dedicated|shared'
+        configuration-value: 'dedicated|shared'
       logicalCpuThreadPinningPolicy:
         schema-version: *any
         schema-location: *any
         platform-id: *generic
         mandatory: *bool
-        configuration-value: r'require|isolate|prefer'
+        configuration-value: 'require|isolate|prefer'
   vduMemRequirements:
       memoryPageSize:
         schema-version: *any
         schema-location: *any
         platform-id: *generic
         mandatory: *bool
-        configuration-value: r'\d+\w*(B|kB|KB|KiB|MB|MiB|GB|GiB|TB|TiB)'
+        configuration-value: '\d+\s*(B|kB|KB|KiB|MB|MiB|GB|GiB|TB|TiB)'
       numberOfPages:
         schema-version: *any
         schema-location: *any
         platform-id: *generic
         mandatory: *bool
-        configuration-value: r'\d+'
+        configuration-value: '\d+'
       memoryAllocationPolicy:
         schema-version: *any
         schema-location: *any
         platform-id: *generic
         mandatory: *bool
-        configuration-value: r'strictLocalAffinity|preferredLocalAffinity'
+        configuration-value: 'strictLocalAffinity|preferredLocalAffinity'
       memoryType:
         schema-version: *any
         schema-location: *any
@@ -132,7 +132,7 @@ schemas:
         schema-location: *any
         platform-id: *generic
         mandatory: *bool
-        configuration-value: r'\d+'
+        configuration-value: '\d+'
       processorCacheAllocationType:
         schema-version: *any
         schema-location: *any
@@ -151,13 +151,13 @@ schemas:
         schema-location: *any
         platform-id: *generic
         mandatory: *bool
-        configuration-value: r'\d+'
+        configuration-value: '\d+'
       storageResilencyMechanism:
         schema-version: *any
         schema-location: *any
         platform-id: *generic
         mandatory: *bool
-        configuration-value: r'erasure|tripleReplication'
+        configuration-value: 'erasure|tripleReplication'
       processorCacheAllocationSize:
         schema-version: *any
         schema-location: *any
@@ -176,13 +176,13 @@ schemas:
         schema-location: *any
         platform-id: *generic
         mandatory: *bool
-        configuration-value: r'\d+'
+        configuration-value: '\d+'
       localNumaMemorySize:
         schema-version: *any
         schema-location: *any
         platform-id: *generic
         mandatory: *bool
-        configuration-value: r'\d+\w*(B|kB|KB|KiB|MB|MiB|GB|GiB|TB|TiB)'
+        configuration-value: '\d+\s*(B|kB|KB|KiB|MB|MiB|GB|GiB|TB|TiB)'
   logicalNodeIoRequirements:
       pciVendorId:
         schema-version: *any
@@ -201,7 +201,7 @@ schemas:
         schema-location: *any
         platform-id: *generic
         mandatory: *bool
-        configuration-value: r'\d+'
+        configuration-value: '\d+'
       pciAddress:
         schema-version: *any
         schema-location: *any
@@ -213,7 +213,7 @@ schemas:
         schema-location: *any
         platform-id: *generic
         mandatory: *bool
-        configuration-value: r'required|notRequired'
+        configuration-value: 'required|notRequired'
   networkInterfaceRequirements:
       nicFeature:
         schema-version: *any
@@ -221,13 +221,13 @@ schemas:
         platform-id: *generic
         mandatory: *bool
         configuration-value: *any
-      dataProcessingAccelerationLibray:
+      dataProcessingAccelerationLibrary:
         schema-version: *any
         schema-location: *any
         platform-id: *generic
         mandatory: *bool
-        configuration-value: r'DPDK'
-      dataProcessingAccelerationLibrayVersion:
+        configuration-value: 'DPDK|dpdk'
+      dataProcessingAccelerationLibraryVersion:
         schema-version: *any
         schema-location: *any
         platform-id: *generic
@@ -238,14 +238,13 @@ schemas:
         schema-location: *any
         platform-id: *generic
         mandatory: *bool
-        configuration-value: r'virtio|PCI-Passthrough|SR-IOV|E1000|RTL8139|PCNET'
+        configuration-value: 'virtio|PCI-Passthrough|SR-IOV|E1000|RTL8139|PCNET'
 mappings:
 # mapping between property value of a tosca node type and the valid hpa schema
 #   type: tosca node type
 #   key: prop1##prop2##...##propN
 #          Property hierachy within that node type. Prefix of '(capability:)'
 #          in propI means the value is from capability instead of property.
-#          Suffix '[] in propI means the value is a list of propI+1.
 #   schema: schema defined in the above schemas section
   - type: tosca.nodes.nfv.Vdu.Compute
     key: capability:virtual_compute##logical_node##logical_node_requirements
@@ -255,13 +254,13 @@ mappings:
     schema: vduCpuRequirements
   - type: tosca.nodes.nfv.Vdu.Compute
     key: capability:virtual_compute##virtual_memory##vdu_memory_requirements
-    schema: vduCpuRequirements
+    schema: vduMemRequirements
   - type: tosca.nodes.nfv.Vdu.VirtualStorage
     key: vdu_storage_requirements
     schema: vduStorageRequirements
   - type: tosca.nodes.nfv.VduCp
-    key: virtual_network_interface_requirements[]##network_interface_requirements##logical_node_requirements
+    key: virtual_network_interface_requirements##network_interface_requirements
     schema: networkInterfaceRequirements
   - type: tosca.nodes.nfv.VduCp
-    key: virtual_network_interface_requirements[]##nic_io_requirements##logical_node_requirements
+    key: virtual_network_interface_requirements##nic_io_requirements##logical_node_requirements
     schema: logicalNodeIoRequirements
diff --git a/vnfsdk_pkgtools/validator/toscaparser_validator.py b/vnfsdk_pkgtools/validator/toscaparser_validator.py
index d1aad30..dfe44b8 100644
--- a/vnfsdk_pkgtools/validator/toscaparser_validator.py
+++ b/vnfsdk_pkgtools/validator/toscaparser_validator.py
@@ -13,9 +13,12 @@
 # under the License.
 #
 
+import functools
+import json
 import logging
 import os
 import pkg_resources
+import re
 
 from toscaparser.common.exception import ValidationError
 from toscaparser.tosca_template import ToscaTemplate
@@ -30,6 +33,10 @@ class HpaSchemaDefError(ValueError):
     pass
 
 
+class HpaValueError(ValueError):
+    pass
+
+
 class ToscaparserValidator(validator.ValidatorBase):
     def __init__(self):
         super(ToscaparserValidator, self).__init__()
@@ -66,5 +73,95 @@ class ToscaparserValidator(validator.ValidatorBase):
         except ValidationError as e:
             LOG.error(e.message)
             raise e
+        self.validate_hpa()
+
+    def is_type(self, node, tosca_type):
+        if node is None:
+            return False
+        elif node.type == tosca_type:
+            return True
+        else:
+            return self.is_type(node.parent_type, tosca_type)
+
+    def extract_value(self, node, key):
+        if node is None:
+            return None
+
+        (cur_key, _, pending) = key.partition('##')
+
+        prefix = None
+        prop = cur_key
+        if ':' in cur_key:
+            (prefix, prop) = cur_key.split(':', 1)
+        if prefix == 'capability':
+            getter = getattr(node, 'get_capability', None)
+            if not getter:
+                raise HpaSchemaDefError("not find capability %s" % prop)
+        elif prefix == 'property' or prefix is None:
+            getter = getattr(node, 'get_property_value', None)
+            if not getter and isinstance(node, dict):
+                getter = getattr(node, 'get')
+        else:
+            raise HpaSchemaDefError("unknown prefix in mapping "
+                                    "key %s" % cur_key)
+        value = getter(prop)
+
+        if not pending:
+            return value
+        elif isinstance(value, list):
+            return list(map(functools.partial(self.extract_value,
+                                              key=pending),
+                            value))
+        else:
+            return self.extract_value(value, pending)
+
+    @staticmethod
+    def validate_value(refkey, hpa_schema, value):
+        if value is None:
+            return
+        if not isinstance(value, dict):
+            msg = "node %s: value %s is not a map of string"
+            raise HpaValueError(msg % (refkey, value))
+        for (key, hpa_value) in value.iteritems():
+            if key not in hpa_schema:
+                msg = "node %s: %s is NOT a valid HPA key"
+                raise HpaValueError(msg  % (refkey, key))
+            try:
+                hpa_dict = json.loads(hpa_value)
+            except:
+                msg = "node %s, HPA key %s: %s is NOT a valid json encoded string"
+                raise HpaValueError(msg % (refkey, key, hpa_value.encode('ascii', 'replace')))
+            if not isinstance(hpa_dict, dict):
+                msg = "node %s, HPA key %s: %s is NOT a valid json encoded string of dict"
+                raise HpaValueError(msg % (refkey, key, hpa_value.encode('ascii', 'replace')))
+            for (attr, val) in hpa_dict.iteritems():
+                if attr not in hpa_schema[key]:
+                    msg = "node %s, HPA key %s: %s is NOT valid HPA attribute"
+                    raise HpaValueError(msg % (refkey, key, attr))
+                attr_schema = hpa_schema[key][attr]
+                if not re.match(attr_schema, str(val)):
+                    msg = ("node %s, HPA key %s, attr %s: %s is not a valid HPA "
+                          "attr value, expected re pattern is %s")
+                    raise HpaValueError(msg % (refkey, key, attr, val.encode('ascii','replace'), attr_schema))
+
+    def validate_hpa_value(self, refkey, hpa_schema, values):
+        if isinstance(values, list):
+            for value in values:
+                self.validate_value(refkey, hpa_schema, value)
+        elif isinstance(values, dict):
+            self.validate_value(refkey, hpa_schema, values)
+
+    def validate_hpa(self):
+        for node in getattr(self.tosca, 'nodetemplates', []):
+            for mapping in self.hpa_mappings:
+                if self.is_type(node, mapping['type']):
+                    value = self.extract_value(node, mapping['key'])
+                    if value:
+                        refkey = node.name + '->' + mapping['key']
+                        LOG.debug("Checking HPA values %s of node %s "
+                                  "against schema %s", value, refkey, mapping['schema'])
+                        self.validate_hpa_value(refkey,
+                                                self.hpa_schemas[mapping['schema']],
+                                                value)
+
 
-        print self.tosca