2 # ============LICENSE_START====================================================
3 # org.onap.vvp/validation-scripts
4 # ===================================================================
5 # Copyright © 2019 AT&T Intellectual Property. All rights reserved.
6 # ===================================================================
8 # Unless otherwise specified, all software contained herein is licensed
9 # under the Apache License, Version 2.0 (the "License");
10 # you may not use this software except in compliance with the License.
11 # You may obtain a copy of the License at
13 # http://www.apache.org/licenses/LICENSE-2.0
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
23 # Unless otherwise specified, all documentation contained herein is licensed
24 # under the Creative Commons License, Attribution 4.0 Intl. (the "License");
25 # you may not use this documentation except in compliance with the License.
26 # You may obtain a copy of the License at
28 # https://creativecommons.org/licenses/by/4.0/
30 # Unless required by applicable law or agreed to in writing, documentation
31 # distributed under the License is distributed on an "AS IS" BASIS,
32 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
33 # See the License for the specific language governing permissions and
34 # limitations under the License.
36 # ============LICENSE_END============================================
45 from tests import cached_yaml as yaml
46 from tests.helpers import load_yaml, get_param
47 from .utils import nested_dict
51 # key = pattern, value = regex compiled from pattern
55 def _get_regex(pattern):
56 """Return a compiled version of pattern.
57 Keep result in _REGEX_CACHE to avoid re-compiling.
59 regex = _REGEX_CACHE.get(pattern, None)
61 regex = re.compile(pattern)
62 _REGEX_CACHE[pattern] = regex
66 class Hashabledict(dict):
68 dicts with the same keys and whose keys have the same values
69 are assigned the same hash.
73 return hash((frozenset(self), frozenset(self.values())))
76 class HeatProcessor(object):
77 """base class for xxxx::xxxx::xxxx processors
80 resource_type = None # string 'xxxx::xxxx::xxxx'
81 re_rids = collections.OrderedDict() # OrderedDict of name: regex
82 # name is a string to name the regex.
83 # regex parses the proper resource id format.
86 def get_param_value(value, withIndex=False):
87 """Return get_param value of `value`
89 if isinstance(value, dict) and len(value) == 1:
90 v = value.get("get_param")
91 if isinstance(v, list) and v:
92 if withIndex and len(v) > 1:
94 if isinstance(idx, dict):
95 idx = idx.get("get_param", idx)
96 v = "{}{}".format(v[0], idx)
104 def get_resource_or_param_value(cls, value):
105 """Return the get_resource or get_param value of `value`
107 if isinstance(value, dict) and len(value) == 1:
108 v = value.get("get_resource") or cls.get_param_value(value)
114 def get_rid_match_tuple(cls, rid):
115 """find the first regex matching `rid` and return the tuple
116 (name, match object) or ('', None) if no match.
118 rid = "" if rid is None else rid
119 for name, regex in cls.re_rids.items():
120 match = regex.match(rid)
126 def get_rid_patterns(cls):
127 """Return OrderedDict of name: friendly regex.pattern
128 "friendly" means the group notation is replaced with
129 braces, and the trailing "$" is removed.
132 nested parentheses in any rid_pattern will break this parser.
133 The final character is ASSUMED to be a dollar sign.
135 friendly_pattern = _get_regex(r"\(\?P<(.*?)>.*?\)")
136 rid_patterns = collections.OrderedDict()
137 for name, regex in cls.re_rids.items():
138 rid_patterns[name] = friendly_pattern.sub(
139 r"{\1}", regex.pattern # replace groups with braces
142 ] # remove trailing $
146 def get_str_replace_name(cls, resource_dict):
147 """Return the name modified by str_replace of `resource_dict`,
148 a resource (i.e. a value in some template's resources).
149 Return None, if there is no name, str_replace, its template,
150 or any missing parameters.
152 str_replace = Heat.nested_get(
153 resource_dict, "properties", "name", "str_replace"
157 template = Heat.nested_get(str_replace, "template")
158 if not isinstance(template, str):
160 params = Heat.nested_get(str_replace, "params", default={})
161 if not isinstance(params, dict):
164 # The user must choose non-overlapping keys for params since they
165 # are replaced in the template in arbitrary order.
167 for key, value in params.items():
168 param = cls.get_param_value(value, withIndex=True)
171 name = name.replace(key, str(param))
175 class CinderVolumeAttachmentProcessor(HeatProcessor):
176 """ Cinder VolumeAttachment
179 resource_type = "OS::Cinder::VolumeAttachment"
182 def get_config(cls, resources):
183 """Return a tuple (va_config, va_count)
184 va_config - Hashabledict of Cinder Volume Attachment config
186 va_count - dict of attachment counts indexed by rid.
188 va_count = collections.defaultdict(int)
189 va_config = Hashabledict()
190 for resource in resources.values():
191 resource_type = nested_dict.get(resource, "type")
192 if resource_type == cls.resource_type:
193 config, rids = cls.get_volume_attachment_config(resource)
195 va_config[rid] = config
197 return va_config, va_count
200 def get_volume_attachment_config(cls, resource):
201 """Returns the cinder volume attachment configuration
202 of `resource` as a tuple (config, rids)
204 - config is a Hashabledict whose keys are the keys of the
205 properties of resource, and whose values are the
206 corresponding property values (nova server resource ids)
207 replaced with the vm-type they reference.
208 - rids is the set of nova server resource ids referenced by
211 config = Hashabledict()
213 for key, value in (resource.get("properties") or {}).items():
214 rid = cls.get_resource_or_param_value(value)
216 name, match = NovaServerProcessor.get_rid_match_tuple(rid)
218 vm_type = match.groupdict()["vm_type"]
219 config[key] = vm_type
224 class ContrailV2NetworkFlavorBaseProcessor(HeatProcessor):
225 """ContrailV2 objects which have network_flavor
228 network_flavor_external = "external"
229 network_flavor_internal = "internal"
230 network_flavor_subint = "subinterface"
233 def get_network_flavor(cls, resource):
234 """Return the network flavor of resource, one of
235 "internal" - get_resource, or get_param contains _int_
236 "subint" - get_param contains _subint_
237 "external" - otherwise
238 None - no parameters found to decide the flavor.
240 resource.properties.virtual_network_refs should be a list.
241 All the parameters in the list should have the same "flavor"
242 so the flavor is determined from the first item.
244 network_flavor = None
245 network_refs = nested_dict.get(resource, "properties", "virtual_network_refs")
246 if network_refs and isinstance(network_refs, list):
247 param = network_refs[0]
248 if isinstance(param, dict):
249 if "get_resource" in param:
250 network_flavor = cls.network_flavor_internal
252 p = param.get("get_param")
253 network_flavor = cls.get_network_format(p)
254 return network_flavor
257 def get_network_format(cls, param):
258 if isinstance(param, str):
259 if "_int_" in param or param.startswith("int_"):
260 return cls.network_flavor_internal
261 elif "_subint_" in param:
262 return cls.network_flavor_subint
264 return cls.network_flavor_external
267 class ContrailV2InstanceIpProcessor(ContrailV2NetworkFlavorBaseProcessor):
268 """ ContrailV2 InstanceIp
271 resource_type = "OS::ContrailV2::InstanceIp"
272 re_rids = collections.OrderedDict(
278 r"_(?P<vm_type_index>\d+)"
280 r"_(?P<network_role>.+)"
282 r"_(?P<vmi_index>\d+)"
293 r"_(?P<vm_type_index>\d+)"
295 r"_(?P<network_role>.+)"
297 r"_(?P<vmi_index>\d+)"
308 r"_(?P<vm_type_index>\d+)"
309 r"_(?P<network_role>.+)"
311 r"_(?P<vmi_index>\d+)"
322 class ContrailV2InterfaceRouteTableProcessor(HeatProcessor):
323 """ ContrailV2 InterfaceRouteTable
326 resource_type = "OS::ContrailV2::InterfaceRouteTable"
329 class ContrailV2NetworkIpamProcessor(HeatProcessor):
330 """ ContrailV2 NetworkIpam
333 resource_type = "OS::ContrailV2::NetworkIpam"
336 class ContrailV2PortTupleProcessor(HeatProcessor):
337 """ ContrailV2 PortTuple
340 resource_type = "OS::ContrailV2::PortTuple"
343 class ContrailV2ServiceHealthCheckProcessor(HeatProcessor):
344 """ ContrailV2 ServiceHealthCheck
347 resource_type = "OS::ContrailV2::ServiceHealthCheck"
350 class ContrailV2ServiceInstanceProcessor(HeatProcessor):
351 """ ContrailV2 ServiceInstance
354 resource_type = "OS::ContrailV2::ServiceInstance"
357 class ContrailV2ServiceInstanceIpProcessor(HeatProcessor):
358 """ ContrailV2 ServiceInstanceIp
361 resource_type = "OS::ContrailV2::ServiceInstanceIp"
364 class ContrailV2ServiceTemplateProcessor(HeatProcessor):
365 """ ContrailV2 ServiceTemplate
368 resource_type = "OS::ContrailV2::ServiceTemplate"
371 class ContrailV2VirtualMachineInterfaceProcessor(ContrailV2NetworkFlavorBaseProcessor):
372 """ ContrailV2 Virtual Machine Interface resource
375 resource_type = "OS::ContrailV2::VirtualMachineInterface"
376 re_rids = collections.OrderedDict(
382 r"_(?P<vm_type_index>\d+)"
384 r"_(?P<network_role>.+)"
386 r"_(?P<vmi_index>\d+)"
394 r"_(?P<vm_type_index>\d+)"
396 r"_(?P<network_role>.+)"
398 r"_(?P<vmi_index>\d+)"
406 r"_(?P<vm_type_index>\d+)"
407 r"_(?P<network_role>.+)"
409 r"_(?P<vmi_index>\d+)"
417 class ContrailV2VirtualNetworkProcessor(HeatProcessor):
418 """ ContrailV2 VirtualNetwork
421 resource_type = "OS::ContrailV2::VirtualNetwork"
422 re_rids = collections.OrderedDict(
424 ("network", _get_regex(r"int" r"_(?P<network_role>.+)" r"_network" r"$")),
425 # ("rvn", _get_regex(r"int" r"_(?P<network_role>.+)" r"_RVN" r"$")),
430 class HeatResourceGroupProcessor(HeatProcessor):
431 """ Heat ResourceGroup
434 resource_type = "OS::Heat::ResourceGroup"
435 re_rids = collections.OrderedDict(
441 r"_(?P<vm_type_index>\d+)"
443 r"_(?P<network_role>.+)"
444 r"_port_(?P<port_index>\d+)"
453 class NeutronNetProcessor(HeatProcessor):
454 """ Neutron Net resource
457 resource_type = "OS::Neutron::Net"
458 re_rids = collections.OrderedDict(
459 [("network", _get_regex(r"int" r"_(?P<network_role>.+)" r"_network" r"$"))]
463 class NeutronPortProcessor(HeatProcessor):
464 """ Neutron Port resource
467 resource_type = "OS::Neutron::Port"
468 re_rids = collections.OrderedDict(
474 r"_(?P<vm_type_index>\d+)"
476 r"_(?P<network_role>.+)"
477 r"_port_(?P<port_index>\d+)"
485 r"_(?P<vm_type_index>\d+)"
486 r"_(?P<network_role>.+)"
487 r"_port_(?P<port_index>\d+)"
495 def uses_sr_iov(cls, resource):
496 """Returns True/False as `resource` is/not
497 An OS::Nova:Port with the property binding:vnic_type
499 resource_properties = nested_dict.get(resource, "properties", default={})
501 nested_dict.get(resource, "type") == cls.resource_type
502 and resource_properties.get("binding:vnic_type", "") == "direct"
509 class NovaServerProcessor(HeatProcessor):
510 """ Nova Server resource
513 resource_type = "OS::Nova::Server"
514 re_rids = collections.OrderedDict(
518 _get_regex(r"(?P<vm_type>.+)" r"_server_(?P<vm_type_index>\d+)" r"$"),
524 def get_flavor(cls, resource):
525 """Return the flavor property of `resource`
527 return cls.get_param_value(nested_dict.get(resource, "properties", "flavor"))
530 def get_image(cls, resource):
531 """Return the image property of `resource`
533 return cls.get_param_value(nested_dict.get(resource, "properties", "image"))
536 def get_network(cls, resource):
537 """Return the network configuration of `resource` as a
538 frozenset of network-roles.
541 networks = nested_dict.get(resource, "properties", "networks")
542 if isinstance(networks, list):
543 for port in networks:
544 value = cls.get_resource_or_param_value(nested_dict.get(port, "port"))
545 name, match = NeutronPortProcessor.get_rid_match_tuple(value)
547 network_role = match.groupdict().get("network_role")
549 network.add(network_role)
550 return frozenset(network)
553 def get_vm_class(cls, resource):
554 """Return the vm_class of `resource`, a Hashabledict (of
555 hashable values) whose keys are only the required keys.
556 Return empty Hashabledict if `resource` is not a NovaServer.
558 vm_class = Hashabledict()
559 resource_type = nested_dict.get(resource, "type")
560 if resource_type == cls.resource_type:
562 flavor=cls.get_flavor(resource),
563 image=cls.get_image(resource),
564 network_role=cls.get_network(resource),
573 filepath - absolute path to template file.
574 envpath - absolute path to environmnt file.
577 type_bool = "boolean"
578 type_boolean = "boolean"
579 type_cdl = "comma_delimited_list"
580 type_comma_delimited_list = "comma_delimited_list"
583 type_number = "number"
585 type_string = "string"
587 def __init__(self, filepath=None, envpath=None):
592 self.heat_template_version = None
593 self.description = None
594 self.parameter_groups = None
595 self.parameters = None
596 self.resources = None
598 self.conditions = None
603 self.load_env(envpath)
604 self.heat_processors = self.get_heat_processors()
608 return "heat_template_version" in self.yml
611 def contrail_resources(self):
612 """This attribute is a dict of Contrail resources.
614 return self.get_resource_by_type(
615 resource_type=ContrailV2VirtualMachineInterfaceProcessor.resource_type
618 def get_all_resources(self, base_dir=None, count=1):
620 Like ``resources``, but this returns all the resources definitions
621 defined in the template, resource groups, and nested YAML files.
623 A special variable will be added to all resource properties (__count__).
624 This will normally be 1, but if the resource is generated by a
625 ResourceGroup **and** an env file is present, then the count will be
626 the value from the env file (assuming this follows standard VNF Heat
629 base_dir = base_dir or self.dirname
631 for r_id, r_data in self.resources.items():
632 r_data["__count__"] = count
633 resources[r_id] = r_data
634 resource = Resource(r_id, r_data)
635 if resource.is_nested():
636 nested_count = resource.get_count(self.env)
637 nested = Heat(os.path.join(base_dir, resource.get_nested_filename()))
638 nested_resources = nested.get_all_resources(count=nested_count)
639 resources.update(nested_resources)
643 def get_heat_processors():
644 """Return a dict, key is resource_type, value is the
645 HeatProcessor subclass whose resource_type is the key.
647 return _HEAT_PROCESSORS
649 def get_resource_by_type(self, resource_type, all_resources=False):
650 """Return dict of resources whose type is `resource_type`.
651 key is resource_id, value is resource.
653 resources = self.get_all_resources() if all_resources else self.resources
656 for rid, resource in resources.items()
657 if self.nested_get(resource, "type") == resource_type
660 def get_rid_match_tuple(self, rid, resource_type):
661 """return get_rid_match_tuple(rid) called on the class
662 corresponding to the given resource_type.
664 processor = self.heat_processors.get(resource_type, HeatProcessor)
665 return processor.get_rid_match_tuple(rid)
667 def get_vm_type(self, rid, resource=None):
668 """return the vm_type
672 resource_type = self.nested_get(resource, "type")
673 match = self.get_rid_match_tuple(rid, resource_type)[1]
674 vm_type = match.groupdict().get("vm_type") if match else None
677 def load(self, filepath):
678 """Load the Heat template given a filepath.
680 self.filepath = filepath
681 self.basename = os.path.basename(self.filepath)
682 self.dirname = os.path.dirname(self.filepath)
683 with open(self.filepath) as fi:
684 self.yml = yaml.load(fi)
685 self.heat_template_version = self.yml.get("heat_template_version", None)
686 self.description = self.yml.get("description", "")
687 self.parameter_groups = self.yml.get("parameter_groups") or {}
688 self.parameters = self.yml.get("parameters") or {}
689 self.resources = self.yml.get("resources") or {}
690 self.outputs = self.yml.get("outputs") or {}
691 self.conditions = self.yml.get("conditions") or {}
693 def load_env(self, envpath):
694 """Load the Environment template given a envpath.
696 self.env = Env(filepath=envpath)
699 def nested_get(dic, *keys, **kwargs):
700 """make utils.nested_dict.get available as a class method.
702 return nested_dict.get(dic, *keys, **kwargs)
705 def neutron_port_resources(self):
706 """This attribute is a dict of Neutron Ports
708 return self.get_resource_by_type(
709 resource_type=NeutronPortProcessor.resource_type
713 def nova_server_resources(self):
714 """This attribute is a dict of Nova Servers
716 return self.get_resource_by_type(
717 resource_type=NovaServerProcessor.resource_type
721 def part_is_in_name(part, name):
723 Return True if any of
724 - name starts with part + '_'
725 - name contains '_' + part + '_'
726 - name ends with '_' + part
730 re.search("(^(%(x)s)_)|(_(%(x)s)_)|(_(%(x)s)$)" % dict(x=part), name)
733 def iter_nested_heat(self):
735 Returns an iterable of tuples (int, heat) where the first parameter is the
736 depth of the nested file and the second item is an instance of Heat
739 def walk_nested(heat, level=1):
740 resources = [Resource(r_id, data) for r_id, data in heat.resources.items()]
741 for resource in resources:
742 if resource.is_nested():
743 nested_path = os.path.join(
744 self.dirname, resource.get_nested_filename()
746 nested_heat = Heat(nested_path)
747 yield level, nested_heat
748 yield from walk_nested(nested_heat, level + 1)
750 yield from walk_nested(self)
753 return "Heat({})".format(self.filepath)
760 """An Environment file
766 class Resource(object):
770 def __init__(self, resource_id=None, resource=None):
771 self.resource_id = resource_id or ""
772 self.resource = resource or {}
773 self.properties = self.resource.get("properties", {})
774 self.resource_type = self.resource.get("type", "")
777 def get_index_var(resource):
778 """Return the index_var for this resource.
780 index_var = nested_dict.get(resource, "properties", "index_var") or "index"
783 def get_nested_filename(self):
784 """Returns the filename of the nested YAML file if the
785 resource is a nested YAML or ResourceDef, returns '' otherwise."""
786 typ = self.resource.get("type", "")
787 if typ == "OS::Heat::ResourceGroup":
788 rd = nested_dict.get(self.resource, "properties", "resource_def")
789 typ = rd.get("type", "") if rd else ""
790 ext = os.path.splitext(typ)[1]
792 if ext == ".yml" or ext == ".yaml":
797 def get_nested_properties(self):
799 Returns {} if not nested
800 Returns resource: properties if nested
801 Returns resource: properties: resource_def: properties if RG
803 if not bool(self.get_nested_filename()):
805 elif self.resource_type == "OS::Heat::ResourceGroup":
806 return nested_dict.get(
807 self.properties, "resource_def", "properties", default={}
810 return self.properties
812 def get_count(self, env):
813 if self.resource_type == "OS::Heat::ResourceGroup":
816 env_params = env.parameters
817 count_param = get_param(self.properties["count"])
818 count_value = env_params.get(count_param) if count_param else 1
820 return int(count_value)
821 except (ValueError, TypeError):
824 "WARNING: Invalid value for count parameter {}. Expected "
825 "an integer, but got {}. Defaulting to 1"
826 ).format(count_param, count_value)
831 def depends_on(self):
833 Returns the list of resources this resource depends on. Always
836 :return: list of all resource IDs this resource depends on. If none,
837 then returns an empty list
839 parents = self.resource.get("depends_on", [])
840 return parents if isinstance(parents, list) else [parents]
843 """Returns True if the resource represents a Nested YAML resource
844 using either type: {filename} or ResourceGroup -> resource_def"""
845 return bool(self.get_nested_filename())
847 def get_nested_yaml(self, base_dir):
848 """If the resource represents a Nested YAML resource, then it
849 returns the loaded YAML. If the resource is not nested or the
850 file cannot be found, then an empty dict is returned"""
851 filename = self.get_nested_filename()
853 file_path = os.path.join(base_dir, filename)
854 return load_yaml(file_path) if os.path.exists(file_path) else {}
859 return "Resource(id={}, type={})".format(self.resource_id, self.resource_type)
865 def get_all_resources(yaml_files):
866 """Return a dict, resource id: resource
867 of the union of resources across all files.
870 for heat_template in yaml_files:
871 heat = Heat(filepath=heat_template)
872 dirname = os.path.dirname(heat_template)
873 resources.update(heat.get_all_resources(dirname))
877 def _get_heat_processors():
878 """Introspect this module and return a
879 dict of all HeatProcessor sub-classes with a (True) resource_type.
880 Key is the resource_type, value is the corrresponding class.
882 mod_classes = inspect.getmembers(sys.modules[__name__], inspect.isclass)
885 for _, c in mod_classes
886 if issubclass(c, HeatProcessor) and c.resource_type
888 return heat_processors
891 _HEAT_PROCESSORS = _get_heat_processors()