From 842888dc28ebccab45e627669f7ee23f04920dc7 Mon Sep 17 00:00:00 2001 From: "Lovett, Trevor" Date: Tue, 8 Oct 2019 13:32:34 -0500 Subject: [PATCH] [VVP] Flag duplicate parameters in .env files Also some minor tweaks to preload generation so it doesn't fail on partial heat templates Change-Id: If39288dde645b0b53a338e7672336807ffa1b6d7 Issue-ID: VVP-284 Signed-off-by: Lovett, Trevor --- ice_validator/heat_requirements.json | 4 +- ice_validator/preload/generator.py | 3 +- ice_validator/preload/model.py | 2 +- .../fail/duplicate_env/duplicate.env | 3 ++ .../fail/duplicate_heat/good_yaml_eg.yaml | 52 ++++++++++++++++++++++ .../fail/{ => invalid_yaml}/fail.yaml | 0 .../pass/good_yaml_eg.env | 2 + ice_validator/tests/test_initial_configuration.py | 28 ++++++------ ice_validator/tests/utils/vm_types.py | 2 +- ice_validator/tests/utils/yaml_custom_utils.py | 47 +++++++++++++------ 10 files changed, 112 insertions(+), 31 deletions(-) create mode 100644 ice_validator/tests/fixtures/test_initial_configuration/fail/duplicate_env/duplicate.env create mode 100644 ice_validator/tests/fixtures/test_initial_configuration/fail/duplicate_heat/good_yaml_eg.yaml rename ice_validator/tests/fixtures/test_initial_configuration/fail/{ => invalid_yaml}/fail.yaml (100%) create mode 100644 ice_validator/tests/fixtures/test_initial_configuration/pass/good_yaml_eg.env diff --git a/ice_validator/heat_requirements.json b/ice_validator/heat_requirements.json index e028715..aabef9a 100644 --- a/ice_validator/heat_requirements.json +++ b/ice_validator/heat_requirements.json @@ -1,5 +1,5 @@ { - "created": "2019-10-07T06:33:21.040671", + "created": "2019-10-08T14:31:11.404157", "current_version": "el alto", "project": "", "versions": { @@ -73163,7 +73163,7 @@ "needs_amount": 813 }, "el alto": { - "created": "2019-10-07T06:33:21.040606", + "created": "2019-10-08T14:31:11.404078", "filters": {}, "filters_amount": 0, "needs": { diff --git a/ice_validator/preload/generator.py b/ice_validator/preload/generator.py index 3da4014..bdd81fa 100644 --- a/ice_validator/preload/generator.py +++ b/ice_validator/preload/generator.py @@ -169,7 +169,8 @@ class AbstractPreloadGenerator(ABC): def generate(self): # handle the base module first print("\nGenerating {} preloads".format(self.format_name())) - self.generate_environments(self.vnf.base_module) + if self.vnf.base_module: + self.generate_environments(self.vnf.base_module) if self.supports_output_passing(): self.vnf.filter_base_outputs() for mod in self.vnf.incremental_modules: diff --git a/ice_validator/preload/model.py b/ice_validator/preload/model.py index c772465..db60438 100644 --- a/ice_validator/preload/model.py +++ b/ice_validator/preload/model.py @@ -236,7 +236,7 @@ class Vnf: @property def base_output_params(self): - return self.base_module.heat.outputs + return self.base_module.heat.outputs if self.base_module else {} def filter_base_outputs(self): non_base_modules = (m for m in self.modules if not m.is_base_module) diff --git a/ice_validator/tests/fixtures/test_initial_configuration/fail/duplicate_env/duplicate.env b/ice_validator/tests/fixtures/test_initial_configuration/fail/duplicate_env/duplicate.env new file mode 100644 index 0000000..c159155 --- /dev/null +++ b/ice_validator/tests/fixtures/test_initial_configuration/fail/duplicate_env/duplicate.env @@ -0,0 +1,3 @@ +parameters: + key: value + key: value2 diff --git a/ice_validator/tests/fixtures/test_initial_configuration/fail/duplicate_heat/good_yaml_eg.yaml b/ice_validator/tests/fixtures/test_initial_configuration/fail/duplicate_heat/good_yaml_eg.yaml new file mode 100644 index 0000000..291a8c3 --- /dev/null +++ b/ice_validator/tests/fixtures/test_initial_configuration/fail/duplicate_heat/good_yaml_eg.yaml @@ -0,0 +1,52 @@ +# -*- coding: utf8 -*- +# ============LICENSE_START======================================================= +# org.onap.vvp/validation-scripts +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# +# Unless otherwise specified, all software contained herein is licensed +# under the Apache License, Version 2.0 (the "License"); +# you may not use this software except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# +# Unless otherwise specified, all documentation contained herein is licensed +# under the Creative Commons License, Attribution 4.0 Intl. (the "License"); +# you may not use this documentation except in compliance with the License. +# You may obtain a copy of the License at +# +# https://creativecommons.org/licenses/by/4.0/ +# +# Unless required by applicable law or agreed to in writing, documentation +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============LICENSE_END============================================ +# +# +heat_template_version: 2015-04-30 + +description: Simple template to deploy a single compute instance + +parameters: + b: + type: string + label: Key Name + description: Name of key-pair to be used for compute instance + + b: + type: string + label: Key Name + description: Name of key-pair to be used for compute instance diff --git a/ice_validator/tests/fixtures/test_initial_configuration/fail/fail.yaml b/ice_validator/tests/fixtures/test_initial_configuration/fail/invalid_yaml/fail.yaml similarity index 100% rename from ice_validator/tests/fixtures/test_initial_configuration/fail/fail.yaml rename to ice_validator/tests/fixtures/test_initial_configuration/fail/invalid_yaml/fail.yaml diff --git a/ice_validator/tests/fixtures/test_initial_configuration/pass/good_yaml_eg.env b/ice_validator/tests/fixtures/test_initial_configuration/pass/good_yaml_eg.env new file mode 100644 index 0000000..d5ecfc7 --- /dev/null +++ b/ice_validator/tests/fixtures/test_initial_configuration/pass/good_yaml_eg.env @@ -0,0 +1,2 @@ +parameters: + key: value diff --git a/ice_validator/tests/test_initial_configuration.py b/ice_validator/tests/test_initial_configuration.py index 654d75d..e4d8b01 100644 --- a/ice_validator/tests/test_initial_configuration.py +++ b/ice_validator/tests/test_initial_configuration.py @@ -65,24 +65,26 @@ def test_00_valid_yaml(filename): ).format(str(e).replace("\n", " ")) +def check_duplicate_keys(yaml_path): + import yaml as normal_yaml + + try: + with open(yaml_path) as fh: + normal_yaml.load(fh, yaml_custom_utils.UniqueKeyLoader) # nosec + except ConstructorError as e: + pytest.fail("{} {}".format(e.problem, e.problem_mark)) + + @pytest.mark.base @validates("R-92635") def test_02_no_duplicate_keys_in_file(yaml_file): - """ - Checks that no duplicate keys exist in a given YAML file. - """ - import yaml as normal_yaml # we can't use the caching version in this test + check_duplicate_keys(yaml_file) - normal_yaml.add_constructor( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, - yaml_custom_utils.raise_duplicates_keys, - ) - try: - with open(yaml_file) as fh: - normal_yaml.safe_load(fh) - except ConstructorError as e: - pytest.fail("{} {}".format(e.problem, e.problem_mark)) +@pytest.mark.base +@validates("R-92635") +def test_02a_no_duplicate_keys_in_env(env_file): + check_duplicate_keys(env_file) @pytest.mark.base diff --git a/ice_validator/tests/utils/vm_types.py b/ice_validator/tests/utils/vm_types.py index e0ea7e7..12fc5a2 100644 --- a/ice_validator/tests/utils/vm_types.py +++ b/ice_validator/tests/utils/vm_types.py @@ -99,7 +99,7 @@ def get_vm_type_for_nova_server(resource): vm_types = get_vm_types_for_resource(resource) # if more than one vm_type was identified, return None - if len(vm_types) > 1: + if not vm_types or len(vm_types) > 1: return None return vm_types.pop() diff --git a/ice_validator/tests/utils/yaml_custom_utils.py b/ice_validator/tests/utils/yaml_custom_utils.py index 6b99dea..597352a 100644 --- a/ice_validator/tests/utils/yaml_custom_utils.py +++ b/ice_validator/tests/utils/yaml_custom_utils.py @@ -38,22 +38,43 @@ # from yaml.constructor import ConstructorError +from yaml.nodes import MappingNode +try: + from yaml import CLoader as SafeLoader +except ImportError: + from yaml import SafeLoader -def raise_duplicates_keys(loader, node, deep=False): - """Raise error when duplicate keys found in yaml file.""" - mapping = {} - for key_node, value_node in node.value: - key = loader.construct_object(key_node, deep=deep) - value = loader.construct_object(value_node, deep=deep) - if key in mapping: +class UniqueKeyLoader(SafeLoader): + def construct_mapping(self, node, deep=False): + if not isinstance(node, MappingNode): raise ConstructorError( - "while constructing a mapping", + None, + None, + "expected a mapping node, but found %s" % node.id, node.start_mark, - "found duplicate key (%s)" % key, - key_node.start_mark, ) - mapping[key] = value - - return loader.construct_mapping(node, deep) + mapping = {} + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + try: + hash(key) + except TypeError as exc: + raise ConstructorError( + "while constructing a mapping", + node.start_mark, + "found unacceptable key (%s)" % exc, + key_node.start_mark, + ) + # check for duplicate keys + if key in mapping: + raise ConstructorError( + "while constructing a mapping", + node.start_mark, + "found duplicate key", + key_node.start_mark, + ) + value = self.construct_object(value_node, deep=deep) + mapping[key] = value + return mapping -- 2.16.6