X-Git-Url: https://gerrit.onap.org/r/gitweb?a=blobdiff_plain;f=ice_validator%2Ftests%2Futils%2Fnested_files.py;h=e1918ad1c67fa171e9c726fbe109403b81cdd9b8;hb=ded5c74ea07eb1541587de1042444fa6b590ddde;hp=cc506d07d64f0c18a69e411c507a0b241a9bf068;hpb=cc21b8b08b6dbcec577bfb26ff397ac899da8002;p=vvp%2Fvalidation-scripts.git

diff --git a/ice_validator/tests/utils/nested_files.py b/ice_validator/tests/utils/nested_files.py
index cc506d0..e1918ad 100644
--- a/ice_validator/tests/utils/nested_files.py
+++ b/ice_validator/tests/utils/nested_files.py
@@ -1,12 +1,12 @@
 # -*- coding: utf8 -*-
-# ============LICENSE_START=======================================================
+# ============LICENSE_START====================================================
 # org.onap.vvp/validation-scripts
 # ===================================================================
 # Copyright © 2017 AT&T Intellectual Property. All rights reserved.
 # ===================================================================
 #
 # Unless otherwise specified, all software contained herein is licensed
-# under the Apache License, Version 2.0 (the “License”);
+# under the Apache License, Version 2.0 (the "License");
 # you may not use this software except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -21,7 +21,7 @@
 #
 #
 # Unless otherwise specified, all documentation contained herein is licensed
-# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# under the Creative Commons License, Attribution 4.0 Intl. (the "License");
 # you may not use this documentation except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -34,109 +34,169 @@
 # limitations under the License.
 #
 # ============LICENSE_END============================================
-#
-# ECOMP is a trademark and service mark of AT&T Intellectual Property.
-#
 
-import yaml
+from functools import lru_cache
+from os import path, listdir
 import re
-from os import path
+from tests import cached_yaml as yaml
 
+from tests.helpers import load_yaml
 
-def get_list_of_nested_files(yml, dirpath):
-    '''
-    return a list of all nested files
-    '''
+MAX_DEPTH = 2
 
-    if not hasattr(yml, 'items'):
-        return []
 
-    nested_files = []
+def check_for_invalid_nesting(  # pylint: disable=too-many-branches
+    yml, yaml_file, dirpath
+):
+    """
+    return a list of all nested files
+    """
+    if not hasattr(yml, "items"):
+        return []
+    invalid_nesting = []
+    p = re.compile("^[A-z]*::[A-z]*::[A-z]*$")
 
-    for k, v in yml.items():
+    for v in yml.values():
         if isinstance(v, dict) and "type" in v:
             t = v["type"]
-            if t.endswith(".yml") or t.endswith(".yaml"):
+            if t.lower().endswith(".yml") or t.lower().endswith(".yaml"):
                 filepath = path.join(dirpath, t)
-                with open(filepath) as fh:
-                    t_yml = yaml.load(fh)
-                nested_files.append(filepath)
-                nested_files.extend(get_list_of_nested_files(t_yml, dirpath))
             elif t == "OS::Heat::ResourceGroup":
-                rdt = v["properties"]["resource_def"]["type"]
-                if rdt.endswith(".yml") or rdt.endswith(".yaml"):
-                    filepath = path.join(dirpath, rdt)
-                    with open(filepath) as fh:
-                        rdt_yml = yaml.load(fh)
-                    nested_files.append(filepath)
-                    nested_files.extend(
-                        get_list_of_nested_files(rdt_yml, dirpath))
+                rd = v["properties"]["resource_def"]
+                if not isinstance(rd, dict) or "type" not in rd:
+                    invalid_nesting.append(yaml_file)
+                    continue
+                elif not p.match(rd["type"]):
+                    filepath = path.join(dirpath, rd["type"])
+                else:
+                    continue
+            else:
+                continue
+            try:
+                with open(filepath) as fh:
+                    yml = yaml.load(fh)
+            except yaml.YAMLError as e:
+                invalid_nesting.append(filepath)
+                print(e)  # pylint: disable=superfluous-parens
+            invalid_nesting.extend(check_for_invalid_nesting(yml, filepath, dirpath))
         if isinstance(v, dict):
-            nested_files.extend(
-                get_list_of_nested_files(v, dirpath))
+            invalid_nesting.extend(check_for_invalid_nesting(v, yaml_file, dirpath))
         elif isinstance(v, list):
             for d in v:
-                nested_files.extend(
-                    get_list_of_nested_files(d, dirpath))
-    return nested_files
+                invalid_nesting.extend(check_for_invalid_nesting(d, yaml_file, dirpath))
+    return invalid_nesting
 
 
-def check_for_invalid_nesting(yml, yaml_file, dirpath):
-    '''
+@lru_cache(maxsize=None)
+def get_list_of_nested_files(yml_path, dirpath):
+    """
     return a list of all nested files
-    '''
-
-    if not hasattr(yml, 'items'):
-        return []
+    """
 
-    invalid_nesting = []
-    p = re.compile('^[A-z]*::[A-z]*::[A-z]*$')
+    yml = load_yaml(yml_path)
+    nested_files = []
+    resources = yml.get("resources") or {}
 
-    for k, v in yml.items():
+    for v in resources.values():
         if isinstance(v, dict) and "type" in v:
             t = v["type"]
-
             if t.endswith(".yml") or t.endswith(".yaml"):
                 filepath = path.join(dirpath, t)
-                try:
-                    with open(filepath) as fh:
-                        t_yml = yaml.load(fh)
-                except Exception as e:
-                    invalid_nesting.append(filepath)
-                    print(e)
-                invalid_nesting.extend(
-                    check_for_invalid_nesting(t_yml,
-                                              filepath,
-                                              dirpath))
+                if path.exists(filepath):
+                    nested_files.append(filepath)
+                    nested_files.extend(get_list_of_nested_files(filepath, dirpath))
             elif t == "OS::Heat::ResourceGroup":
-                rd = v["properties"]["resource_def"]
-                if not isinstance(rd, dict):
-                    invalid_nesting.append(yaml_file)
-                elif "type" not in rd:
-                    invalid_nesting.append(yaml_file)
-                elif not p.match(rd["type"]) and not \
-                    (rd["type"].endswith(".yml")
-                     or rd["type"].endswith(".yaml")):
-                    filepath = path.join(dirpath, rd["type"])
-                    try:
-                        with open(filepath) as fh:
-                            rdt_yml = yaml.load(fh)
-                    except Exception as e:
-                        invalid_nesting.append(filepath)
-                        print(e)
-                    invalid_nesting.extend(
-                        check_for_invalid_nesting(rdt_yml,
-                                                  filepath,
-                                                  dirpath))
-        if isinstance(v, dict):
-            invalid_nesting.extend(
-                check_for_invalid_nesting(v,
-                                          yaml_file,
-                                          dirpath))
-        elif isinstance(v, list):
-            for d in v:
-                invalid_nesting.extend(
-                    check_for_invalid_nesting(d,
-                                              yaml_file,
-                                              dirpath))
-    return invalid_nesting
+                rdt = v.get("properties", {}).get("resource_def", {}).get("type", None)
+                if rdt and (rdt.endswith(".yml") or rdt.endswith(".yaml")):
+                    filepath = path.join(dirpath, rdt)
+                    if path.exists(filepath):
+                        nested_files.append(filepath)
+                        nested_files.extend(get_list_of_nested_files(filepath, dirpath))
+    return nested_files
+
+
+def get_resourcegroup_nested_files(yml, dirpath):
+    """
+    return a dict.
+    key: key in yml which references a nested ResourceGroup file.
+        (resource->type is ResourceGroup
+            and resource->properties->resource_def->type is a yaml file)
+    value: the nested file name.
+
+    The keys are assumed to be unique across files.
+    A separate test checks for that.
+    """
+
+    if not hasattr(yml, "get"):
+        return {}
+
+    nested_files = {}
+    for rid, r in yml.get("resources", {}).items():
+        if isinstance(r, dict) and "type" in r:
+            t = r["type"]
+            nested_file = None
+            if t == "OS::Heat::ResourceGroup":
+                rdt = r.get("properties", {}).get("resource_def", {}).get("type", None)
+                if rdt and (rdt.endswith(".yml") or rdt.endswith(".yaml")):
+                    nested_file = rdt
+            if nested_file:
+                filepath = path.join(dirpath, nested_file)
+                if path.exists(filepath):
+                    nested_files[rid] = nested_file
+    return nested_files
+
+
+def get_type_nested_files(yml, dirpath):
+    """
+    return a dict.
+    key: key in yml which references a nested type file.
+        (the resource "type" is a yaml file.)
+    value: the nested file name.
+
+    The keys are assumed to be unique across files.
+    A separate test checks for that.
+    """
+
+    if not hasattr(yml, "get"):
+        return {}
+
+    nested_files = {}
+    for rid, r in yml.get("resources", {}).items():
+        if isinstance(r, dict) and "type" in r:
+            t = r["type"]
+            nested_file = None
+            if t.endswith(".yml") or t.endswith(".yaml"):
+                nested_file = t
+            if nested_file:
+                filepath = path.join(dirpath, nested_file)
+                if path.exists(filepath):
+                    nested_files[rid] = nested_file
+    return nested_files
+
+
+def get_nested_files(filenames):
+    """
+    returns all the nested files for a set of filenames
+    """
+    nested_files = []
+    for filename in filenames:
+        if file_is_a_nested_template(filename):
+            nested_files.append(filename)
+    return nested_files
+
+
+@lru_cache(maxsize=None)
+def file_is_a_nested_template(file):
+    directory = path.dirname(file)
+    nested_files = []
+    for filename in listdir(directory):
+        if filename.endswith(".yaml") or filename.endswith(".yml"):
+            filename = "{}/{}".format(directory, filename)
+            try:
+                nested_files.extend(
+                    get_list_of_nested_files(filename, path.dirname(filename))
+                )
+            except yaml.YAMLError as e:
+                print(e)  # pylint: disable=superfluous-parens
+                continue
+    return file in nested_files