2 # ============LICENSE_START====================================================
3 # org.onap.vvp/validation-scripts
4 # ===================================================================
5 # Copyright © 2019 AT&T Intellectual Property. All rights reserved.
6 # ===================================================================
8 # Unless otherwise specified, all software contained herein is licensed
9 # under the Apache License, Version 2.0 (the "License");
10 # you may not use this software except in compliance with the License.
11 # You may obtain a copy of the License at
13 # http://www.apache.org/licenses/LICENSE-2.0
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
23 # Unless otherwise specified, all documentation contained herein is licensed
24 # under the Creative Commons License, Attribution 4.0 Intl. (the "License");
25 # you may not use this documentation except in compliance with the License.
26 # You may obtain a copy of the License at
28 # https://creativecommons.org/licenses/by/4.0/
30 # Unless required by applicable law or agreed to in writing, documentation
31 # distributed under the License is distributed on an "AS IS" BASIS,
32 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
33 # See the License for the specific language governing permissions and
34 # limitations under the License.
36 # ============LICENSE_END============================================
47 from tests import cached_yaml as yaml
48 from tests.helpers import load_yaml
49 from .utils import nested_dict
53 # key = pattern, value = regex compiled from pattern
57 def _get_regex(pattern):
58 """Return a compiled version of pattern.
59 Keep result in _REGEX_CACHE to avoid re-compiling.
61 regex = _REGEX_CACHE.get(pattern, None)
63 regex = re.compile(pattern)
64 _REGEX_CACHE[pattern] = regex
68 class Hashabledict(dict):
70 dicts with the same keys and whose keys have the same values
71 are assigned the same hash.
75 return hash((frozenset(self), frozenset(self.values())))
78 class HeatProcessor(object):
79 """base class for xxxx::xxxx::xxxx processors
82 resource_type = None # string 'xxxx::xxxx::xxxx'
83 re_rids = collections.OrderedDict() # OrderedDict of name: regex
84 # name is a string to name the regex.
85 # regex parses the proper resource id format.
88 def get_param_value(value):
89 """Return get_param value of `value`
91 if isinstance(value, dict) and len(value) == 1:
92 v = value.get("get_param")
93 if isinstance(v, list) and v:
100 def get_resource_or_param_value(cls, value):
101 """Return the get_resource or get_param value of `value`
103 if isinstance(value, dict) and len(value) == 1:
104 v = value.get("get_resource") or cls.get_param_value(value)
110 def get_rid_match_tuple(cls, rid):
111 """find the first regex matching `rid` and return the tuple
112 (name, match object) or ('', None) if no match.
114 rid = "" if rid is None else rid
115 for name, regex in cls.re_rids.items():
116 match = regex.match(rid)
122 def get_rid_patterns(cls):
123 """Return OrderedDict of name: friendly regex.pattern
124 "friendly" means the group notation is replaced with
125 braces, and the trailing "$" is removed.
128 nested parentheses in any rid_pattern will break this parser.
129 The final character is ASSUMED to be a dollar sign.
131 friendly_pattern = _get_regex(r"\(\?P<(.*?)>.*?\)")
132 rid_patterns = collections.OrderedDict()
133 for name, regex in cls.re_rids.items():
134 rid_patterns[name] = friendly_pattern.sub(
135 r"{\1}", regex.pattern # replace groups with braces
138 ] # remove trailing $
142 def get_str_replace_name(cls, resource_dict):
143 """Return the name modified by str_replace of `resource_dict`,
144 a resource (i.e. a value in some template's resources).
145 Return None, if there is no name, str_replace, its template,
146 or any missing parameters.
148 str_replace = Heat.nested_get(
149 resource_dict, "properties", "name", "str_replace"
153 template = Heat.nested_get(str_replace, "template")
154 if not isinstance(template, str):
156 params = Heat.nested_get(str_replace, "params", default={})
157 if not isinstance(params, dict):
160 # The user must choose non-overlapping keys for params since they
161 # are replaced in the template in arbitrary order.
163 for key, value in params.items():
164 param = cls.get_param_value(value)
167 name = name.replace(key, str(param))
171 class CinderVolumeAttachmentProcessor(HeatProcessor):
172 """ Cinder VolumeAttachment
175 resource_type = "OS::Cinder::VolumeAttachment"
178 def get_config(cls, resources):
179 """Return a tuple (va_config, va_count)
180 va_config - Hashabledict of Cinder Volume Attachment config
182 va_count - dict of attachment counts indexed by rid.
184 va_count = collections.defaultdict(int)
185 va_config = Hashabledict()
186 for resource in resources.values():
187 resource_type = nested_dict.get(resource, "type")
188 if resource_type == cls.resource_type:
189 config, rids = cls.get_volume_attachment_config(resource)
191 va_config[rid] = config
193 return va_config, va_count
196 def get_volume_attachment_config(cls, resource):
197 """Returns the cinder volume attachment configuration
198 of `resource` as a tuple (config, rids)
200 - config is a Hashabledict whose keys are the keys of the
201 properties of resource, and whose values are the
202 corresponding property values (nova server resource ids)
203 replaced with the vm-type they reference.
204 - rids is the set of nova server resource ids referenced by
207 config = Hashabledict()
209 for key, value in (resource.get("properties") or {}).items():
210 rid = cls.get_resource_or_param_value(value)
212 name, match = NovaServerProcessor.get_rid_match_tuple(rid)
214 vm_type = match.groupdict()["vm_type"]
215 config[key] = vm_type
220 class ContrailV2NetworkFlavorBaseProcessor(HeatProcessor):
221 """ContrailV2 objects which have network_flavor
224 network_flavor_external = "external"
225 network_flavor_internal = "internal"
226 network_flavor_subint = "subint"
229 def get_network_flavor(cls, resource):
230 """Return the network flavor of resource, one of
231 "internal" - get_resource, or get_param contains _int_
232 "subint" - get_param contains _subint_
233 "external" - otherwise
234 None - no parameters found to decide the flavor.
236 resource.properties.virtual_network_refs should be a list.
237 All the parameters in the list should have the same "flavor"
238 so the flavor is determined from the first item.
240 network_flavor = None
241 network_refs = nested_dict.get(resource, "properties", "virtual_network_refs")
242 if network_refs and isinstance(network_refs, list):
243 param = network_refs[0]
244 if isinstance(param, dict):
245 if "get_resource" in param:
246 network_flavor = cls.network_flavor_internal
248 p = param.get("get_param")
249 if isinstance(p, str):
250 if "_int_" in p or p.startswith("int_"):
251 network_flavor = cls.network_flavor_internal
252 elif "_subint_" in p:
253 network_flavor = cls.network_flavor_subint
255 network_flavor = cls.network_flavor_external
256 return network_flavor
259 class ContrailV2InstanceIpProcessor(ContrailV2NetworkFlavorBaseProcessor):
260 """ ContrailV2 InstanceIp
263 resource_type = "OS::ContrailV2::InstanceIp"
264 re_rids = collections.OrderedDict(
270 r"_(?P<vm_type_index>\d+)"
272 r"_(?P<network_role>.+)"
274 r"_(?P<vmi_index>\d+)"
284 r"_(?P<vm_type_index>\d+)"
286 r"_(?P<network_role>.+)"
288 r"_(?P<vmi_index>\d+)"
298 r"_(?P<vm_type_index>\d+)"
300 r"_(?P<network_role>.+)"
302 r"_(?P<vmi_index>\d+)"
312 r"_(?P<vm_type_index>\d+)"
314 r"_(?P<network_role>.+)"
316 r"_(?P<vmi_index>\d+)"
326 r"_(?P<vm_type_index>\d+)"
327 r"_(?P<network_role>.+)"
329 r"_(?P<vmi_index>\d+)"
339 r"_(?P<vm_type_index>\d+)"
340 r"_(?P<network_role>.+)"
342 r"_(?P<vmi_index>\d+)"
352 class ContrailV2InterfaceRouteTableProcessor(HeatProcessor):
353 """ ContrailV2 InterfaceRouteTable
356 resource_type = "OS::ContrailV2::InterfaceRouteTable"
359 class ContrailV2NetworkIpamProcessor(HeatProcessor):
360 """ ContrailV2 NetworkIpam
363 resource_type = "OS::ContrailV2::NetworkIpam"
366 class ContrailV2PortTupleProcessor(HeatProcessor):
367 """ ContrailV2 PortTuple
370 resource_type = "OS::ContrailV2::PortTuple"
373 class ContrailV2ServiceHealthCheckProcessor(HeatProcessor):
374 """ ContrailV2 ServiceHealthCheck
377 resource_type = "OS::ContrailV2::ServiceHealthCheck"
380 class ContrailV2ServiceInstanceProcessor(HeatProcessor):
381 """ ContrailV2 ServiceInstance
384 resource_type = "OS::ContrailV2::ServiceInstance"
387 class ContrailV2ServiceInstanceIpProcessor(HeatProcessor):
388 """ ContrailV2 ServiceInstanceIp
391 resource_type = "OS::ContrailV2::ServiceInstanceIp"
394 class ContrailV2ServiceTemplateProcessor(HeatProcessor):
395 """ ContrailV2 ServiceTemplate
398 resource_type = "OS::ContrailV2::ServiceTemplate"
401 class ContrailV2VirtualMachineInterfaceProcessor(ContrailV2NetworkFlavorBaseProcessor):
402 """ ContrailV2 Virtual Machine Interface resource
405 resource_type = "OS::ContrailV2::VirtualMachineInterface"
406 re_rids = collections.OrderedDict(
412 r"_(?P<vm_type_index>\d+)"
414 r"_(?P<network_role>.+)"
416 r"_(?P<vmi_index>\d+)"
424 r"_(?P<vm_type_index>\d+)"
426 r"_(?P<network_role>.+)"
428 r"_(?P<vmi_index>\d+)"
436 r"_(?P<vm_type_index>\d+)"
437 r"_(?P<network_role>.+)"
439 r"_(?P<vmi_index>\d+)"
447 class ContrailV2VirtualNetworkProcessor(HeatProcessor):
448 """ ContrailV2 VirtualNetwork
451 resource_type = "OS::ContrailV2::VirtualNetwork"
452 re_rids = collections.OrderedDict(
454 ("network", _get_regex(r"int" r"_(?P<network_role>.+)" r"_network" r"$")),
455 ("rvn", _get_regex(r"int" r"_(?P<network_role>.+)" r"_RVN" r"$")),
460 class HeatResourceGroupProcessor(HeatProcessor):
461 """ Heat ResourceGroup
464 resource_type = "OS::Heat::ResourceGroup"
465 re_rids = collections.OrderedDict(
471 r"_(?P<vm_type_index>\d+)"
473 r"_(?P<network_role>.+)"
474 r"_port_(?P<port_index>\d+)"
483 class NeutronNetProcessor(HeatProcessor):
484 """ Neutron Net resource
487 resource_type = "OS::Neutron::Net"
488 re_rids = collections.OrderedDict(
489 [("network", _get_regex(r"int" r"_(?P<network_role>.+)" r"_network" r"$"))]
493 class NeutronPortProcessor(HeatProcessor):
494 """ Neutron Port resource
497 resource_type = "OS::Neutron::Port"
498 re_rids = collections.OrderedDict(
504 r"_(?P<vm_type_index>\d+)"
506 r"_(?P<network_role>.+)"
507 r"_port_(?P<port_index>\d+)"
515 r"_(?P<vm_type_index>\d+)"
516 r"_(?P<network_role>.+)"
517 r"_port_(?P<port_index>\d+)"
526 r"_(?P<network_role>.+)"
527 r"_floating_ip_(?P<index>\d+)"
536 r"_(?P<network_role>.+)"
537 r"_floating_v6_ip_(?P<index>\d+)"
545 def uses_sr_iov(cls, resource):
546 """Returns True/False as `resource` is/not
547 An OS::Nova:Port with the property binding:vnic_type
549 return nested_dict.get(
551 ) == cls.resource_type and "binding:vnic_type" in nested_dict.get(
552 resource, "properties", default={}
556 class NovaServerProcessor(HeatProcessor):
557 """ Nova Server resource
560 resource_type = "OS::Nova::Server"
561 re_rids = collections.OrderedDict(
565 _get_regex(r"(?P<vm_type>.+)" r"_server_(?P<vm_type_index>\d+)" r"$"),
571 def get_flavor(cls, resource):
572 """Return the flavor property of `resource`
574 return cls.get_param_value(nested_dict.get(resource, "properties", "flavor"))
577 def get_image(cls, resource):
578 """Return the image property of `resource`
580 return cls.get_param_value(nested_dict.get(resource, "properties", "image"))
583 def get_network(cls, resource):
584 """Return the network configuration of `resource` as a
585 frozenset of network-roles.
588 networks = nested_dict.get(resource, "properties", "networks")
589 if isinstance(networks, list):
590 for port in networks:
591 value = cls.get_resource_or_param_value(nested_dict.get(port, "port"))
592 name, match = NeutronPortProcessor.get_rid_match_tuple(value)
594 network_role = match.groupdict().get("network_role")
596 network.add(network_role)
597 return frozenset(network)
600 def get_vm_class(cls, resource):
601 """Return the vm_class of `resource`, a Hashabledict (of
602 hashable values) whose keys are only the required keys.
603 Return empty Hashabledict if `resource` is not a NovaServer.
605 vm_class = Hashabledict()
606 resource_type = nested_dict.get(resource, "type")
607 if resource_type == cls.resource_type:
609 flavor=cls.get_flavor(resource),
610 image=cls.get_image(resource),
611 networks=cls.get_network(resource),
620 filepath - absolute path to template file.
621 envpath - absolute path to environmnt file.
624 type_bool = "boolean"
625 type_boolean = "boolean"
626 type_cdl = "comma_delimited_list"
627 type_comma_delimited_list = "comma_delimited_list"
630 type_number = "number"
632 type_string = "string"
634 def __init__(self, filepath=None, envpath=None):
639 self.heat_template_version = None
640 self.description = None
641 self.parameter_groups = None
642 self.parameters = None
643 self.resources = None
645 self.conditions = None
650 self.load_env(envpath)
651 self.heat_processors = self.get_heat_processors()
654 def contrail_resources(self):
655 """This attribute is a dict of Contrail resources.
657 return self.get_resource_by_type(
658 resource_type=ContrailV2VirtualMachineInterfaceProcessor.resource_type
661 def get_all_resources(self, base_dir):
664 but this returns all the resources definitions
665 defined in the template, resource groups, and nested YAML files.
668 for r_id, r_data in self.resources.items():
669 resources[r_id] = r_data
670 resource = Resource(r_id, r_data)
671 if resource.is_nested():
672 nested = Heat(os.path.join(base_dir, resource.get_nested_filename()))
673 resources.update(nested.get_all_resources(base_dir))
677 def get_heat_processors():
678 """Return a dict, key is resource_type, value is the
679 HeatProcessor subclass whose resource_type is the key.
681 return _HEAT_PROCESSORS
683 def get_resource_by_type(self, resource_type):
684 """Return dict of resources whose type is `resource_type`.
685 key is resource_id, value is resource.
689 for rid, resource in self.resources.items()
690 if self.nested_get(resource, "type") == resource_type
693 def get_rid_match_tuple(self, rid, resource_type):
694 """return get_rid_match_tuple(rid) called on the class
695 corresponding to the given resource_type.
697 processor = self.heat_processors.get(resource_type, HeatProcessor)
698 return processor.get_rid_match_tuple(rid)
700 def get_vm_type(self, rid, resource=None):
701 """return the vm_type
705 resource_type = self.nested_get(resource, "type")
706 match = self.get_rid_match_tuple(rid, resource_type)[1]
707 vm_type = match.groupdict().get("vm_type") if match else None
710 def load(self, filepath):
711 """Load the Heat template given a filepath.
713 self.filepath = filepath
714 self.basename = os.path.basename(self.filepath)
715 self.dirname = os.path.dirname(self.filepath)
716 with open(self.filepath) as fi:
717 self.yml = yaml.load(fi)
718 self.heat_template_version = self.yml.get("heat_template_version", None)
719 self.description = self.yml.get("description", "")
720 self.parameter_groups = self.yml.get("parameter_groups") or {}
721 self.parameters = self.yml.get("parameters") or {}
722 self.resources = self.yml.get("resources") or {}
723 self.outputs = self.yml.get("outputs") or {}
724 self.conditions = self.yml.get("conditions") or {}
726 def load_env(self, envpath):
727 """Load the Environment template given a envpath.
729 self.env = Env(filepath=envpath)
732 def nested_get(dic, *keys, **kwargs):
733 """make utils.nested_dict.get available as a class method.
735 return nested_dict.get(dic, *keys, **kwargs)
738 def neutron_port_resources(self):
739 """This attribute is a dict of Neutron Ports
741 return self.get_resource_by_type(
742 resource_type=NeutronPortProcessor.resource_type
746 def nova_server_resources(self):
747 """This attribute is a dict of Nova Servers
749 return self.get_resource_by_type(
750 resource_type=NovaServerProcessor.resource_type
754 def part_is_in_name(part, name):
756 Return True if any of
757 - name starts with part + '_'
758 - name contains '_' + part + '_'
759 - name ends with '_' + part
763 re.search("(^(%(x)s)_)|(_(%(x)s)_)|(_(%(x)s)$)" % dict(x=part), name)
768 """An Environment file
774 class Resource(object):
778 def __init__(self, resource_id=None, resource=None):
779 self.resource_id = resource_id or ""
780 self.resource = resource or {}
781 self.properties = self.resource.get("properties", {})
782 self.resource_type = self.resource.get("type", "")
785 def get_index_var(resource):
786 """Return the index_var for this resource.
788 index_var = nested_dict.get(resource, "properties", "index_var") or "index"
791 def get_nested_filename(self):
792 """Returns the filename of the nested YAML file if the
793 resource is a nested YAML or ResourceDef, returns '' otherwise."""
794 typ = self.resource.get("type", "")
795 if typ == "OS::Heat::ResourceGroup":
796 rd = nested_dict.get(self.resource, "properties", "resource_def")
797 typ = rd.get("type", "") if rd else ""
798 ext = os.path.splitext(typ)[1]
800 if ext == ".yml" or ext == ".yaml":
805 def get_nested_properties(self):
807 Returns {} if not nested
808 Returns resource: properties if nested
809 Returns resource: properties: resource_def: properties if RG
811 if not bool(self.get_nested_filename()):
813 elif self.resource_type == "OS::Heat::ResourceGroup":
814 return nested_dict.get(
815 self.properties, "resource_def", "properties", default={}
818 return self.properties
821 def depends_on(self):
823 Returns the list of resources this resource depends on. Always
826 :return: list of all resource IDs this resource depends on. If none,
827 then returns an empty list
829 parents = self.resource.get("depends_on", [])
830 return parents if isinstance(parents, list) else [parents]
833 """Returns True if the resource represents a Nested YAML resource
834 using either type: {filename} or ResourceGroup -> resource_def"""
835 return bool(self.get_nested_filename())
837 def get_nested_yaml(self, base_dir):
838 """If the resource represents a Nested YAML resource, then it
839 returns the loaded YAML. If the resource is not nested or the
840 file cannot be found, then an empty dict is returned"""
841 filename = self.get_nested_filename()
843 file_path = os.path.join(base_dir, filename)
844 return load_yaml(file_path) if os.path.exists(file_path) else {}
849 def get_all_resources(yaml_files):
850 """Return a dict, resource id: resource
851 of the union of resources across all files.
854 for heat_template in yaml_files:
855 heat = Heat(filepath=heat_template)
856 dirname = os.path.dirname(heat_template)
857 resources.update(heat.get_all_resources(dirname))
861 def _get_heat_processors():
862 """Introspect this module and return a
863 dict of all HeatProcessor sub-classes with a (True) resource_type.
864 Key is the resource_type, value is the corrresponding class.
866 mod_classes = inspect.getmembers(sys.modules[__name__], inspect.isclass)
869 for _, c in mod_classes
870 if issubclass(c, HeatProcessor) and c.resource_type
872 return heat_processors
875 _HEAT_PROCESSORS = _get_heat_processors()