2 # ============LICENSE_START====================================================
3 # org.onap.vvp/validation-scripts
4 # ===================================================================
5 # Copyright © 2019 AT&T Intellectual Property. All rights reserved.
6 # ===================================================================
8 # Unless otherwise specified, all software contained herein is licensed
9 # under the Apache License, Version 2.0 (the "License");
10 # you may not use this software except in compliance with the License.
11 # You may obtain a copy of the License at
13 # http://www.apache.org/licenses/LICENSE-2.0
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
23 # Unless otherwise specified, all documentation contained herein is licensed
24 # under the Creative Commons License, Attribution 4.0 Intl. (the "License");
25 # you may not use this documentation except in compliance with the License.
26 # You may obtain a copy of the License at
28 # https://creativecommons.org/licenses/by/4.0/
30 # Unless required by applicable law or agreed to in writing, documentation
31 # distributed under the License is distributed on an "AS IS" BASIS,
32 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
33 # See the License for the specific language governing permissions and
34 # limitations under the License.
36 # ============LICENSE_END============================================
38 # ECOMP is a trademark and service mark of AT&T Intellectual Property.
48 from tests import cached_yaml as yaml
49 from tests.helpers import load_yaml
50 from .utils import nested_dict
54 # key = pattern, value = regex compiled from pattern
58 def _get_regex(pattern):
59 """Return a compiled version of pattern.
60 Keep result in _REGEX_CACHE to avoid re-compiling.
62 regex = _REGEX_CACHE.get(pattern, None)
64 regex = re.compile(pattern)
65 _REGEX_CACHE[pattern] = regex
69 class Hashabledict(dict):
71 dicts with the same keys and whose keys have the same values
72 are assigned the same hash.
76 return hash((frozenset(self), frozenset(self.values())))
79 class HeatProcessor(object):
80 """base class for xxxx::xxxx::xxxx processors
83 resource_type = None # string 'xxxx::xxxx::xxxx'
84 re_rids = collections.OrderedDict() # OrderedDict of name: regex
85 # name is a string to name the regex.
86 # regex parses the proper resource id format.
89 def get_param_value(value):
90 """Return get_param value of `value`
92 if isinstance(value, dict) and len(value) == 1:
93 v = value.get("get_param")
94 if isinstance(v, list) and v:
101 def get_resource_or_param_value(cls, value):
102 """Return the get_resource or get_param value of `value`
104 if isinstance(value, dict) and len(value) == 1:
105 v = value.get("get_resource") or cls.get_param_value(value)
111 def get_rid_match_tuple(cls, rid):
112 """find the first regex matching `rid` and return the tuple
113 (name, match object) or ('', None) if no match.
115 rid = "" if rid is None else rid
116 for name, regex in cls.re_rids.items():
117 match = regex.match(rid)
123 def get_rid_patterns(cls):
124 """Return OrderedDict of name: friendly regex.pattern
125 "friendly" means the group notation is replaced with
126 braces, and the trailing "$" is removed.
129 nested parentheses in any rid_pattern will break this parser.
130 The final character is ASSUMED to be a dollar sign.
132 friendly_pattern = _get_regex(r"\(\?P<(.*?)>.*?\)")
133 rid_patterns = collections.OrderedDict()
134 for name, regex in cls.re_rids.items():
135 rid_patterns[name] = friendly_pattern.sub(
136 r"{\1}", regex.pattern # replace groups with braces
139 ] # remove trailing $
143 def get_str_replace_name(cls, resource_dict):
144 """Return the name modified by str_replace of `resource_dict`,
145 a resource (i.e. a value in some template's resources).
146 Return None, if there is no name, str_replace, its template,
147 or any missing parameters.
149 str_replace = Heat.nested_get(
150 resource_dict, "properties", "name", "str_replace"
154 template = Heat.nested_get(str_replace, "template")
155 if not isinstance(template, str):
157 params = Heat.nested_get(str_replace, "params", default={})
158 if not isinstance(params, dict):
161 # The user must choose non-overlapping keys for params since they
162 # are replaced in the template in arbitrary order.
164 for key, value in params.items():
165 param = cls.get_param_value(value)
168 name = name.replace(key, str(param))
172 class CinderVolumeAttachmentProcessor(HeatProcessor):
173 """ Cinder VolumeAttachment
176 resource_type = "OS::Cinder::VolumeAttachment"
179 def get_config(cls, resources):
180 """Return a tuple (va_config, va_count)
181 va_config - Hashabledict of Cinder Volume Attachment config
183 va_count - dict of attachment counts indexed by rid.
185 va_count = collections.defaultdict(int)
186 va_config = Hashabledict()
187 for resource in resources.values():
188 resource_type = nested_dict.get(resource, "type")
189 if resource_type == cls.resource_type:
190 config, rids = cls.get_volume_attachment_config(resource)
192 va_config[rid] = config
194 return va_config, va_count
197 def get_volume_attachment_config(cls, resource):
198 """Returns the cinder volume attachment configuration
199 of `resource` as a tuple (config, rids)
201 - config is a Hashabledict whose keys are the keys of the
202 properties of resource, and whose values are the
203 corresponding property values (nova server resource ids)
204 replaced with the vm-type they reference.
205 - rids is the set of nova server resource ids referenced by
208 config = Hashabledict()
210 for key, value in (resource.get("properties") or {}).items():
211 rid = cls.get_resource_or_param_value(value)
213 name, match = NovaServerProcessor.get_rid_match_tuple(rid)
215 vm_type = match.groupdict()["vm_type"]
216 config[key] = vm_type
221 class ContrailV2NetworkFlavorBaseProcessor(HeatProcessor):
222 """ContrailV2 objects which have network_flavor
225 network_flavor_external = "external"
226 network_flavor_internal = "internal"
227 network_flavor_subint = "subint"
230 def get_network_flavor(cls, resource):
231 """Return the network flavor of resource, one of
232 "internal" - get_resource, or get_param contains _int_
233 "subint" - get_param contains _subint_
234 "external" - otherwise
235 None - no parameters found to decide the flavor.
237 resource.properties.virtual_network_refs should be a list.
238 All the parameters in the list should have the same "flavor"
239 so the flavor is determined from the first item.
241 network_flavor = None
242 network_refs = nested_dict.get(resource, "properties", "virtual_network_refs")
243 if network_refs and isinstance(network_refs, list):
244 param = network_refs[0]
245 if isinstance(param, dict):
246 if "get_resource" in param:
247 network_flavor = cls.network_flavor_internal
249 p = param.get("get_param")
250 if isinstance(p, str):
251 if "_int_" in p or p.startswith("int_"):
252 network_flavor = cls.network_flavor_internal
253 elif "_subint_" in p:
254 network_flavor = cls.network_flavor_subint
256 network_flavor = cls.network_flavor_external
257 return network_flavor
260 class ContrailV2InstanceIpProcessor(ContrailV2NetworkFlavorBaseProcessor):
261 """ ContrailV2 InstanceIp
264 resource_type = "OS::ContrailV2::InstanceIp"
265 re_rids = collections.OrderedDict(
271 r"_(?P<vm_type_index>\d+)"
273 r"_(?P<network_role>.+)"
275 r"_(?P<vmi_index>\d+)"
285 r"_(?P<vm_type_index>\d+)"
287 r"_(?P<network_role>.+)"
289 r"_(?P<vmi_index>\d+)"
299 r"_(?P<vm_type_index>\d+)"
301 r"_(?P<network_role>.+)"
303 r"_(?P<vmi_index>\d+)"
313 r"_(?P<vm_type_index>\d+)"
315 r"_(?P<network_role>.+)"
317 r"_(?P<vmi_index>\d+)"
327 r"_(?P<vm_type_index>\d+)"
328 r"_(?P<network_role>.+)"
330 r"_(?P<vmi_index>\d+)"
340 r"_(?P<vm_type_index>\d+)"
341 r"_(?P<network_role>.+)"
343 r"_(?P<vmi_index>\d+)"
353 class ContrailV2InterfaceRouteTableProcessor(HeatProcessor):
354 """ ContrailV2 InterfaceRouteTable
357 resource_type = "OS::ContrailV2::InterfaceRouteTable"
360 class ContrailV2NetworkIpamProcessor(HeatProcessor):
361 """ ContrailV2 NetworkIpam
364 resource_type = "OS::ContrailV2::NetworkIpam"
367 class ContrailV2PortTupleProcessor(HeatProcessor):
368 """ ContrailV2 PortTuple
371 resource_type = "OS::ContrailV2::PortTuple"
374 class ContrailV2ServiceHealthCheckProcessor(HeatProcessor):
375 """ ContrailV2 ServiceHealthCheck
378 resource_type = "OS::ContrailV2::ServiceHealthCheck"
381 class ContrailV2ServiceInstanceProcessor(HeatProcessor):
382 """ ContrailV2 ServiceInstance
385 resource_type = "OS::ContrailV2::ServiceInstance"
388 class ContrailV2ServiceInstanceIpProcessor(HeatProcessor):
389 """ ContrailV2 ServiceInstanceIp
392 resource_type = "OS::ContrailV2::ServiceInstanceIp"
395 class ContrailV2ServiceTemplateProcessor(HeatProcessor):
396 """ ContrailV2 ServiceTemplate
399 resource_type = "OS::ContrailV2::ServiceTemplate"
402 class ContrailV2VirtualMachineInterfaceProcessor(ContrailV2NetworkFlavorBaseProcessor):
403 """ ContrailV2 Virtual Machine Interface resource
406 resource_type = "OS::ContrailV2::VirtualMachineInterface"
407 re_rids = collections.OrderedDict(
413 r"_(?P<vm_type_index>\d+)"
415 r"_(?P<network_role>.+)"
417 r"_(?P<vmi_index>\d+)"
425 r"_(?P<vm_type_index>\d+)"
427 r"_(?P<network_role>.+)"
429 r"_(?P<vmi_index>\d+)"
437 r"_(?P<vm_type_index>\d+)"
438 r"_(?P<network_role>.+)"
440 r"_(?P<vmi_index>\d+)"
448 class ContrailV2VirtualNetworkProcessor(HeatProcessor):
449 """ ContrailV2 VirtualNetwork
452 resource_type = "OS::ContrailV2::VirtualNetwork"
453 re_rids = collections.OrderedDict(
455 ("network", _get_regex(r"int" r"_(?P<network_role>.+)" r"_network" r"$")),
456 ("rvn", _get_regex(r"int" r"_(?P<network_role>.+)" r"_RVN" r"$")),
461 class HeatResourceGroupProcessor(HeatProcessor):
462 """ Heat ResourceGroup
465 resource_type = "OS::Heat::ResourceGroup"
466 re_rids = collections.OrderedDict(
472 r"_(?P<vm_type_index>\d+)"
474 r"_(?P<network_role>.+)"
475 r"_port_(?P<port_index>\d+)"
484 class NeutronNetProcessor(HeatProcessor):
485 """ Neutron Net resource
488 resource_type = "OS::Neutron::Net"
489 re_rids = collections.OrderedDict(
490 [("network", _get_regex(r"int" r"_(?P<network_role>.+)" r"_network" r"$"))]
494 class NeutronPortProcessor(HeatProcessor):
495 """ Neutron Port resource
498 resource_type = "OS::Neutron::Port"
499 re_rids = collections.OrderedDict(
505 r"_(?P<vm_type_index>\d+)"
507 r"_(?P<network_role>.+)"
508 r"_port_(?P<port_index>\d+)"
516 r"_(?P<vm_type_index>\d+)"
517 r"_(?P<network_role>.+)"
518 r"_port_(?P<port_index>\d+)"
527 r"_(?P<network_role>.+)"
528 r"_floating_ip_(?P<index>\d+)"
537 r"_(?P<network_role>.+)"
538 r"_floating_v6_ip_(?P<index>\d+)"
546 def uses_sr_iov(cls, resource):
547 """Returns True/False as `resource` is/not
548 An OS::Nova:Port with the property binding:vnic_type
550 return nested_dict.get(
552 ) == cls.resource_type and "binding:vnic_type" in nested_dict.get(
553 resource, "properties", default={}
557 class NovaServerProcessor(HeatProcessor):
558 """ Nova Server resource
561 resource_type = "OS::Nova::Server"
562 re_rids = collections.OrderedDict(
566 _get_regex(r"(?P<vm_type>.+)" r"_server_(?P<vm_type_index>\d+)" r"$"),
572 def get_flavor(cls, resource):
573 """Return the flavor property of `resource`
575 return cls.get_param_value(nested_dict.get(resource, "properties", "flavor"))
578 def get_image(cls, resource):
579 """Return the image property of `resource`
581 return cls.get_param_value(nested_dict.get(resource, "properties", "image"))
584 def get_network(cls, resource):
585 """Return the network configuration of `resource` as a
586 frozenset of network-roles.
589 networks = nested_dict.get(resource, "properties", "networks")
590 if isinstance(networks, list):
591 for port in networks:
592 value = cls.get_resource_or_param_value(nested_dict.get(port, "port"))
593 name, match = NeutronPortProcessor.get_rid_match_tuple(value)
595 network_role = match.groupdict().get("network_role")
597 network.add(network_role)
598 return frozenset(network)
601 def get_vm_class(cls, resource):
602 """Return the vm_class of `resource`, a Hashabledict (of
603 hashable values) whose keys are only the required keys.
604 Return empty Hashabledict if `resource` is not a NovaServer.
606 vm_class = Hashabledict()
607 resource_type = nested_dict.get(resource, "type")
608 if resource_type == cls.resource_type:
610 flavor=cls.get_flavor(resource),
611 image=cls.get_image(resource),
612 networks=cls.get_network(resource),
621 filepath - absolute path to template file.
622 envpath - absolute path to environmnt file.
625 type_bool = "boolean"
626 type_boolean = "boolean"
627 type_cdl = "comma_delimited_list"
628 type_comma_delimited_list = "comma_delimited_list"
631 type_number = "number"
633 type_string = "string"
635 def __init__(self, filepath=None, envpath=None):
640 self.heat_template_version = None
641 self.description = None
642 self.parameter_groups = None
643 self.parameters = None
644 self.resources = None
646 self.conditions = None
651 self.load_env(envpath)
652 self.heat_processors = self.get_heat_processors()
655 def contrail_resources(self):
656 """This attribute is a dict of Contrail resources.
658 return self.get_resource_by_type(
659 resource_type=ContrailV2VirtualMachineInterfaceProcessor.resource_type
662 def get_all_resources(self, base_dir):
665 but this returns all the resources definitions
666 defined in the template, resource groups, and nested YAML files.
669 for r_id, r_data in self.resources.items():
670 resources[r_id] = r_data
671 resource = Resource(r_id, r_data)
672 if resource.is_nested():
673 nested = Heat(os.path.join(base_dir, resource.get_nested_filename()))
674 resources.update(nested.get_all_resources(base_dir))
678 def get_heat_processors():
679 """Return a dict, key is resource_type, value is the
680 HeatProcessor subclass whose resource_type is the key.
682 return _HEAT_PROCESSORS
684 def get_resource_by_type(self, resource_type):
685 """Return dict of resources whose type is `resource_type`.
686 key is resource_id, value is resource.
690 for rid, resource in self.resources.items()
691 if self.nested_get(resource, "type") == resource_type
694 def get_rid_match_tuple(self, rid, resource_type):
695 """return get_rid_match_tuple(rid) called on the class
696 corresponding to the given resource_type.
698 processor = self.heat_processors.get(resource_type, HeatProcessor)
699 return processor.get_rid_match_tuple(rid)
701 def get_vm_type(self, rid, resource=None):
702 """return the vm_type
706 resource_type = self.nested_get(resource, "type")
707 match = self.get_rid_match_tuple(rid, resource_type)[1]
708 vm_type = match.groupdict().get("vm_type") if match else None
711 def load(self, filepath):
712 """Load the Heat template given a filepath.
714 self.filepath = filepath
715 self.basename = os.path.basename(self.filepath)
716 self.dirname = os.path.dirname(self.filepath)
717 with open(self.filepath) as fi:
718 self.yml = yaml.load(fi)
719 self.heat_template_version = self.yml.get("heat_template_version", None)
720 self.description = self.yml.get("description", "")
721 self.parameter_groups = self.yml.get("parameter_groups") or {}
722 self.parameters = self.yml.get("parameters") or {}
723 self.resources = self.yml.get("resources") or {}
724 self.outputs = self.yml.get("outputs") or {}
725 self.conditions = self.yml.get("conditions") or {}
727 def load_env(self, envpath):
728 """Load the Environment template given a envpath.
730 self.env = Env(filepath=envpath)
733 def nested_get(dic, *keys, **kwargs):
734 """make utils.nested_dict.get available as a class method.
736 return nested_dict.get(dic, *keys, **kwargs)
739 def neutron_port_resources(self):
740 """This attribute is a dict of Neutron Ports
742 return self.get_resource_by_type(
743 resource_type=NeutronPortProcessor.resource_type
747 def nova_server_resources(self):
748 """This attribute is a dict of Nova Servers
750 return self.get_resource_by_type(
751 resource_type=NovaServerProcessor.resource_type
755 def part_is_in_name(part, name):
757 Return True if any of
758 - name starts with part + '_'
759 - name contains '_' + part + '_'
760 - name ends with '_' + part
764 re.search("(^(%(x)s)_)|(_(%(x)s)_)|(_(%(x)s)$)" % dict(x=part), name)
769 """An Environment file
775 class Resource(object):
779 def __init__(self, resource_id=None, resource=None):
780 self.resource_id = resource_id or ""
781 self.resource = resource or {}
782 self.properties = self.resource.get("properties", {})
783 self.resource_type = self.resource.get("type", "")
786 def get_index_var(resource):
787 """Return the index_var for this resource.
789 index_var = nested_dict.get(resource, "properties", "index_var") or "index"
792 def get_nested_filename(self):
793 """Returns the filename of the nested YAML file if the
794 resource is a nested YAML or ResourceDef, returns '' otherwise."""
795 typ = self.resource.get("type", "")
796 if typ == "OS::Heat::ResourceGroup":
797 rd = nested_dict.get(self.resource, "properties", "resource_def")
798 typ = rd.get("type", "") if rd else ""
799 ext = os.path.splitext(typ)[1]
801 if ext == ".yml" or ext == ".yaml":
806 def get_nested_properties(self):
808 Returns {} if not nested
809 Returns resource: properties if nested
810 Returns resource: properties: resource_def: properties if RG
812 if not bool(self.get_nested_filename()):
814 elif self.resource_type == "OS::Heat::ResourceGroup":
815 return nested_dict.get(
816 self.properties, "resource_def", "properties", default={}
819 return self.properties
822 def depends_on(self):
824 Returns the list of resources this resource depends on. Always
827 :return: list of all resource IDs this resource depends on. If none,
828 then returns an empty list
830 parents = self.resource.get("depends_on", [])
831 return parents if isinstance(parents, list) else [parents]
834 """Returns True if the resource represents a Nested YAML resource
835 using either type: {filename} or ResourceGroup -> resource_def"""
836 return bool(self.get_nested_filename())
838 def get_nested_yaml(self, base_dir):
839 """If the resource represents a Nested YAML resource, then it
840 returns the loaded YAML. If the resource is not nested or the
841 file cannot be found, then an empty dict is returned"""
842 filename = self.get_nested_filename()
844 file_path = os.path.join(base_dir, filename)
845 return load_yaml(file_path) if os.path.exists(file_path) else {}
850 def get_all_resources(yaml_files):
851 """Return a dict, resource id: resource
852 of the union of resources across all files.
855 for heat_template in yaml_files:
856 heat = Heat(filepath=heat_template)
857 dirname = os.path.dirname(heat_template)
858 resources.update(heat.get_all_resources(dirname))
862 def _get_heat_processors():
863 """Introspect this module and return a
864 dict of all HeatProcessor sub-classes with a (True) resource_type.
865 Key is the resource_type, value is the corrresponding class.
867 mod_classes = inspect.getmembers(sys.modules[__name__], inspect.isclass)
870 for _, c in mod_classes
871 if issubclass(c, HeatProcessor) and c.resource_type
873 return heat_processors
876 _HEAT_PROCESSORS = _get_heat_processors()