ice_validator/tests/structures.py

   1 # -*- coding: utf8 -*-
   2 # ============LICENSE_START====================================================
   3 # org.onap.vvp/validation-scripts
   4 # ===================================================================
   5 # Copyright © 2017 AT&T Intellectual Property. All rights reserved.
   6 # ===================================================================
   7 #
   8 # Unless otherwise specified, all software contained herein is licensed
   9 # under the Apache License, Version 2.0 (the "License");
  10 # you may not use this software except in compliance with the License.
  11 # You may obtain a copy of the License at
  12 #
  13 #             http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS,
  17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18 # See the License for the specific language governing permissions and
  19 # limitations under the License.
  20 #
  21 #
  22 #
  23 # Unless otherwise specified, all documentation contained herein is licensed
  24 # under the Creative Commons License, Attribution 4.0 Intl. (the "License");
  25 # you may not use this documentation except in compliance with the License.
  26 # You may obtain a copy of the License at
  27 #
  28 #             https://creativecommons.org/licenses/by/4.0/
  29 #
  30 # Unless required by applicable law or agreed to in writing, documentation
  31 # distributed under the License is distributed on an "AS IS" BASIS,
  32 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  33 # See the License for the specific language governing permissions and
  34 # limitations under the License.
  35 #
  36 # ============LICENSE_END============================================
  37 #
  38
  39 """structures
  40 """
  41 import sys
  42
  43
  44 import collections
  45 import inspect
  46 import os
  47 import re
  48
  49 from tests import cached_yaml as yaml
  50 from tests.helpers import load_yaml
  51
  52 from .utils import nested_dict
  53
  54 VERSION = "3.5.0"
  55
  56 # key = pattern, value = regex compiled from pattern
  57 _REGEX_CACHE = {}
  58
  59
  60 def _get_regex(pattern):
  61     """Return a compiled version of pattern.
  62     Keep result in _REGEX_CACHE to avoid re-compiling.
  63     """
  64     regex = _REGEX_CACHE.get(pattern, None)
  65     if regex is None:
  66         regex = re.compile(pattern)
  67         _REGEX_CACHE[pattern] = regex
  68     return regex
  69
  70
  71 class HeatObject(object):
  72     """base class for xxxx::xxxx::xxxx objects
  73     """
  74
  75     resource_type = None
  76
  77     def __init__(self):
  78         self.re_rids = self.get_re_rids()
  79
  80     @staticmethod
  81     def get_re_rids():
  82         """Return OrderedDict of name: regex
  83         Each regex parses the proper format for a given rid
  84         (resource id).
  85         """
  86         return collections.OrderedDict()
  87
  88     def get_rid_match_tuple(self, rid):
  89         """find the first regex matching `rid` and return the tuple
  90         (name, match object) or ('', None) if no match.
  91         """
  92         for name, regex in self.re_rids.items():
  93             match = regex.match(rid)
  94             if match:
  95                 return name, match
  96         return "", None
  97
  98     def get_rid_patterns(self):
  99         """Return OrderedDict of name: friendly regex.pattern
 100         "friendly" means the group notation is replaced with
 101         braces, and the trailing "$" is removed.
 102
 103         NOTE
 104         nested parentheses in any rid_pattern will break this parser.
 105         The final character is ASSUMED to be a dollar sign.
 106         """
 107         friendly_pattern = _get_regex(r"\(\?P<(.*?)>.*?\)")
 108         rid_patterns = collections.OrderedDict()
 109         for name, regex in self.re_rids.items():
 110             rid_patterns[name] = friendly_pattern.sub(
 111                 r"{\1}", regex.pattern  # replace groups with braces
 112             )[
 113                 :-1
 114             ]  # remove trailing $
 115         return rid_patterns
 116
 117
 118 class ContrailV2NetworkHeatObject(HeatObject):
 119     """ContrailV2 objects which have network_flavor
 120     """
 121
 122     network_flavor_external = "external"
 123     network_flavor_internal = "internal"
 124     network_flavor_subint = "subint"
 125
 126     def get_network_flavor(self, resource):
 127         """Return the network flavor of resource, one of
 128         "internal" - get_resource, or get_param contains _int_
 129         "subint" - get_param contains _subint_
 130         "external" - otherwise
 131         None - no parameters found to decide the flavor.
 132
 133         resource.properties.virtual_network_refs should be a list.
 134         All the parameters in the list should have the same "flavor"
 135         so the flavor is determined from the first item.
 136         """
 137         network_flavor = None
 138         network_refs = nested_dict.get(resource, "properties", "virtual_network_refs")
 139         if network_refs and isinstance(network_refs, list):
 140             param = network_refs[0]
 141             if isinstance(param, dict):
 142                 if "get_resource" in param:
 143                     network_flavor = self.network_flavor_internal
 144                 else:
 145                     p = param.get("get_param")
 146                     if isinstance(p, str):
 147                         if "_int_" in p or p.startswith("int_"):
 148                             network_flavor = self.network_flavor_internal
 149                         elif "_subint_" in p:
 150                             network_flavor = self.network_flavor_subint
 151                         else:
 152                             network_flavor = self.network_flavor_external
 153         return network_flavor
 154
 155
 156 class ContrailV2InstanceIp(ContrailV2NetworkHeatObject):
 157     """ ContrailV2 InstanceIp
 158     """
 159
 160     resource_type = "OS::ContrailV2::InstanceIp"
 161
 162     def get_re_rids(self):
 163         """Return OrderedDict of name: regex
 164         """
 165         return collections.OrderedDict(
 166             [
 167                 (
 168                     "int_ip",
 169                     _get_regex(
 170                         r"(?P<vm_type>.+)"
 171                         r"_(?P<vm_type_index>\d+)"
 172                         r"_int"
 173                         r"_(?P<network_role>.+)"
 174                         r"_vmi"
 175                         r"_(?P<vmi_index>\d+)"
 176                         r"_IP"
 177                         r"_(?P<index>\d+)"
 178                         r"$"
 179                     ),
 180                 ),
 181                 (
 182                     "int_v6_ip",
 183                     _get_regex(
 184                         r"(?P<vm_type>.+)"
 185                         r"_(?P<vm_type_index>\d+)"
 186                         r"_int"
 187                         r"_(?P<network_role>.+)"
 188                         r"_vmi"
 189                         r"_(?P<vmi_index>\d+)"
 190                         r"_v6_IP"
 191                         r"_(?P<index>\d+)"
 192                         r"$"
 193                     ),
 194                 ),
 195                 (
 196                     "subint_ip",
 197                     _get_regex(
 198                         r"(?P<vm_type>.+)"
 199                         r"_(?P<vm_type_index>\d+)"
 200                         r"_subint"
 201                         r"_(?P<network_role>.+)"
 202                         r"_vmi"
 203                         r"_(?P<vmi_index>\d+)"
 204                         r"_IP"
 205                         r"_(?P<index>\d+)"
 206                         r"$"
 207                     ),
 208                 ),
 209                 (
 210                     "subint_v6_ip",
 211                     _get_regex(
 212                         r"(?P<vm_type>.+)"
 213                         r"_(?P<vm_type_index>\d+)"
 214                         r"_subint"
 215                         r"_(?P<network_role>.+)"
 216                         r"_vmi"
 217                         r"_(?P<vmi_index>\d+)"
 218                         r"_v6_IP"
 219                         r"_(?P<index>\d+)"
 220                         r"$"
 221                     ),
 222                 ),
 223                 (
 224                     "ip",
 225                     _get_regex(
 226                         r"(?P<vm_type>.+)"
 227                         r"_(?P<vm_type_index>\d+)"
 228                         r"_(?P<network_role>.+)"
 229                         r"_vmi"
 230                         r"_(?P<vmi_index>\d+)"
 231                         r"_IP"
 232                         r"_(?P<index>\d+)"
 233                         r"$"
 234                     ),
 235                 ),
 236                 (
 237                     "v6_ip",
 238                     _get_regex(
 239                         r"(?P<vm_type>.+)"
 240                         r"_(?P<vm_type_index>\d+)"
 241                         r"_(?P<network_role>.+)"
 242                         r"_vmi"
 243                         r"_(?P<vmi_index>\d+)"
 244                         r"_v6_IP"
 245                         r"_(?P<index>\d+)"
 246                         r"$"
 247                     ),
 248                 ),
 249             ]
 250         )
 251
 252
 253 class ContrailV2InterfaceRouteTable(HeatObject):
 254     """ ContrailV2 InterfaceRouteTable
 255     """
 256
 257     resource_type = "OS::ContrailV2::InterfaceRouteTable"
 258
 259
 260 class ContrailV2NetworkIpam(HeatObject):
 261     """ ContrailV2 NetworkIpam
 262     """
 263
 264     resource_type = "OS::ContrailV2::NetworkIpam"
 265
 266
 267 class ContrailV2PortTuple(HeatObject):
 268     """ ContrailV2 PortTuple
 269     """
 270
 271     resource_type = "OS::ContrailV2::PortTuple"
 272
 273
 274 class ContrailV2ServiceHealthCheck(HeatObject):
 275     """ ContrailV2 ServiceHealthCheck
 276     """
 277
 278     resource_type = "OS::ContrailV2::ServiceHealthCheck"
 279
 280
 281 class ContrailV2ServiceInstance(HeatObject):
 282     """ ContrailV2 ServiceInstance
 283     """
 284
 285     resource_type = "OS::ContrailV2::ServiceInstance"
 286
 287
 288 class ContrailV2ServiceInstanceIp(HeatObject):
 289     """ ContrailV2 ServiceInstanceIp
 290     """
 291
 292     resource_type = "OS::ContrailV2::ServiceInstanceIp"
 293
 294
 295 class ContrailV2ServiceTemplate(HeatObject):
 296     """ ContrailV2 ServiceTemplate
 297     """
 298
 299     resource_type = "OS::ContrailV2::ServiceTemplate"
 300
 301
 302 class ContrailV2VirtualMachineInterface(ContrailV2NetworkHeatObject):
 303     """ ContrailV2 Virtual Machine Interface resource
 304     """
 305
 306     resource_type = "OS::ContrailV2::VirtualMachineInterface"
 307
 308     def get_re_rids(self):
 309         """Return OrderedDict of name: regex
 310         """
 311         return collections.OrderedDict(
 312             [
 313                 (
 314                     "vmi_internal",
 315                     _get_regex(
 316                         r"(?P<vm_type>.+)"
 317                         r"_(?P<vm_type_index>\d+)"
 318                         r"_int"
 319                         r"_(?P<network_role>.+)"
 320                         r"_vmi"
 321                         r"_(?P<vmi_index>\d+)"
 322                         r"$"
 323                     ),
 324                 ),
 325                 (
 326                     "vmi_subint",
 327                     _get_regex(
 328                         r"(?P<vm_type>.+)"
 329                         r"_(?P<vm_type_index>\d+)"
 330                         r"_subint"
 331                         r"_(?P<network_role>.+)"
 332                         r"_vmi"
 333                         r"_(?P<vmi_index>\d+)"
 334                         r"$"
 335                     ),
 336                 ),
 337                 (
 338                     "vmi_external",
 339                     _get_regex(
 340                         r"(?P<vm_type>.+)"
 341                         r"_(?P<vm_type_index>\d+)"
 342                         r"_(?P<network_role>.+)"
 343                         r"_vmi"
 344                         r"_(?P<vmi_index>\d+)"
 345                         r"$"
 346                     ),
 347                 ),
 348             ]
 349         )
 350
 351
 352 class ContrailV2VirtualNetwork(HeatObject):
 353     """ ContrailV2 VirtualNetwork
 354     """
 355
 356     resource_type = "OS::ContrailV2::VirtualNetwork"
 357
 358     def get_re_rids(self):
 359         """Return OrderedDict of name: regex
 360         """
 361         return collections.OrderedDict(
 362             [
 363                 (
 364                     "network",
 365                     _get_regex(r"int" r"_(?P<network_role>.+)" r"_network" r"$"),
 366                 ),
 367                 ("rvn", _get_regex(r"int" r"_(?P<network_role>.+)" r"_RVN" r"$")),
 368             ]
 369         )
 370
 371
 372 class NeutronNet(HeatObject):
 373     """ Neutron Net resource
 374     """
 375
 376     resource_type = "OS::Neutron::Net"
 377
 378     def get_re_rids(self):
 379         """Return OrderedDict of name: regex
 380         """
 381         return collections.OrderedDict(
 382             [("network", _get_regex(r"int" r"_(?P<network_role>.+)" r"_network" r"$"))]
 383         )
 384
 385
 386 class NeutronPort(HeatObject):
 387     """ Neutron Port resource
 388     """
 389
 390     resource_type = "OS::Neutron::Port"
 391
 392     def get_re_rids(self):
 393         """Return OrderedDict of name: regex
 394         """
 395         return collections.OrderedDict(
 396             [
 397                 (
 398                     "internal_port",
 399                     _get_regex(
 400                         r"(?P<vm_type>.+)"
 401                         r"_(?P<vm_type_index>\d+)"
 402                         r"_int"
 403                         r"_(?P<network_role>.+)"
 404                         r"_port_(?P<port_index>\d+)"
 405                         r"$"
 406                     ),
 407                 ),
 408                 (
 409                     "port",
 410                     _get_regex(
 411                         r"(?P<vm_type>.+)"
 412                         r"_(?P<vm_type_index>\d+)"
 413                         r"_(?P<network_role>.+)"
 414                         r"_port_(?P<port_index>\d+)"
 415                         r"$"
 416                     ),
 417                 ),
 418                 (
 419                     "floating_ip",
 420                     _get_regex(
 421                         r"reserve_port"
 422                         r"_(?P<vm_type>.+)"
 423                         r"_(?P<network_role>.+)"
 424                         r"_floating_ip_(?P<index>\d+)"
 425                         r"$"
 426                     ),
 427                 ),
 428                 (
 429                     "floating_v6_ip",
 430                     _get_regex(
 431                         r"reserve_port"
 432                         r"_(?P<vm_type>.+)"
 433                         r"_(?P<network_role>.+)"
 434                         r"_floating_v6_ip_(?P<index>\d+)"
 435                         r"$"
 436                     ),
 437                 ),
 438             ]
 439         )
 440
 441
 442 class NovaServer(HeatObject):
 443     """ Nova Server resource
 444     """
 445
 446     resource_type = "OS::Nova::Server"
 447
 448     def get_re_rids(self):
 449         """Return OrderedDict of name: regex
 450         """
 451         return collections.OrderedDict(
 452             [
 453                 (
 454                     "server",
 455                     _get_regex(
 456                         r"(?P<vm_type>.+)" r"_server_(?P<vm_type_index>\d+)" r"$"
 457                     ),
 458                 )
 459             ]
 460         )
 461
 462
 463 class Heat(object):
 464     """A Heat template.
 465     filepath - absolute path to template file.
 466     envpath - absolute path to environmnt file.
 467     """
 468
 469     type_cdl = "comma_delimited_list"
 470     type_num = "number"
 471     type_str = "string"
 472
 473     def __init__(self, filepath=None, envpath=None):
 474         self.filepath = None
 475         self.basename = None
 476         self.dirname = None
 477         self.yml = None
 478         self.heat_template_version = None
 479         self.description = None
 480         self.parameter_groups = None
 481         self.parameters = None
 482         self.resources = None
 483         self.outputs = None
 484         self.conditions = None
 485         if filepath:
 486             self.load(filepath)
 487         self.env = None
 488         if envpath:
 489             self.load_env(envpath)
 490         self.heat_objects = self.get_heat_objects()
 491
 492     @property
 493     def contrail_resources(self):
 494         """This attribute is a dict of Contrail resources.
 495         """
 496         return self.get_resource_by_type(
 497             resource_type=ContrailV2VirtualMachineInterface.resource_type
 498         )
 499
 500     @staticmethod
 501     def get_heat_objects():
 502         """Return a dict, key is resource_type, value is the
 503         HeatObject subclass whose resource_type is the key.
 504         """
 505         return _HEAT_OBJECTS
 506
 507     def get_resource_by_type(self, resource_type):
 508         """Return dict of resources whose type is `resource_type`.
 509         key is resource_id, value is resource.
 510         """
 511         return {
 512             rid: resource
 513             for rid, resource in self.resources.items()
 514             if self.nested_get(resource, "type") == resource_type
 515         }
 516
 517     def get_rid_match_tuple(self, rid, resource_type):
 518         """return get_rid_match_tuple(rid) called on the class
 519         corresponding to the given resource_type.
 520         """
 521         hoc = self.heat_objects.get(resource_type, HeatObject)
 522         return hoc().get_rid_match_tuple(rid)
 523
 524     def get_vm_type(self, rid, resource=None):
 525         """return the vm_type
 526         """
 527         if resource is None:
 528             resource = self
 529         resource_type = self.nested_get(resource, "type")
 530         match = self.get_rid_match_tuple(rid, resource_type)[1]
 531         vm_type = match.groupdict().get("vm_type") if match else None
 532         return vm_type
 533
 534     def load(self, filepath):
 535         """Load the Heat template given a filepath.
 536         """
 537         self.filepath = filepath
 538         self.basename = os.path.basename(self.filepath)
 539         self.dirname = os.path.dirname(self.filepath)
 540         with open(self.filepath) as fi:
 541             self.yml = yaml.load(fi)
 542         self.heat_template_version = self.yml.get("heat_template_version", None)
 543         self.description = self.yml.get("description", "")
 544         self.parameter_groups = self.yml.get("parameter_groups", {})
 545         self.parameters = self.yml.get("parameters") or {}
 546         self.resources = self.yml.get("resources", {})
 547         self.outputs = self.yml.get("outputs", {})
 548         self.conditions = self.yml.get("conditions", {})
 549
 550     def get_all_resources(self, base_dir):
 551         """
 552         Like ``resources``, but this returns all the resources definitions
 553         defined in the template, resource groups, and nested YAML files.
 554         """
 555         resources = {}
 556         for r_id, r_data in self.resources.items():
 557             resources[r_id] = r_data
 558             resource = Resource(r_id, r_data)
 559             if resource.is_nested():
 560                 nested = Heat(os.path.join(base_dir, resource.get_nested_filename()))
 561                 resources.update(nested.get_all_resources(base_dir))
 562         return resources
 563
 564     def load_env(self, envpath):
 565         """
 566         Load the Environment template given a envpath.
 567         """
 568         self.env = Env(filepath=envpath)
 569
 570     @staticmethod
 571     def nested_get(dic, *keys, **kwargs):
 572         """make utils.nested_dict.get available as a class method.
 573         """
 574         return nested_dict.get(dic, *keys, **kwargs)
 575
 576     @property
 577     def neutron_port_resources(self):
 578         """This attribute is a dict of Neutron Ports
 579         """
 580         return self.get_resource_by_type(resource_type=NeutronPort.resource_type)
 581
 582     @property
 583     def nova_server_resources(self):
 584         """This attribute is a dict of Nova Servers
 585         """
 586         return self.get_resource_by_type(resource_type=NovaServer.resource_type)
 587
 588     @staticmethod
 589     def part_is_in_name(part, name):
 590         """
 591         Return True if any of
 592         - name starts with part + '_'
 593         - name contains '_' + part + '_'
 594         - name ends with '_' + part
 595         False otherwise
 596         """
 597         return bool(
 598             re.search("(^(%(x)s)_)|(_(%(x)s)_)|(_(%(x)s)$)" % dict(x=part), name)
 599         )
 600
 601
 602 class Env(Heat):
 603     """An Environment file
 604     """
 605
 606     pass
 607
 608
 609 class Resource(object):
 610     """A Resource
 611     """
 612
 613     def __init__(self, resource_id=None, resource=None):
 614         self.resource_id = resource_id or ""
 615         self.resource = resource or {}
 616         self.properties = self.resource.get("properties", {})
 617         self.resource_type = resource.get("type", "")
 618
 619     @staticmethod
 620     def get_index_var(resource):
 621         """Return the index_var for this resource.
 622         """
 623         index_var = nested_dict.get(resource, "properties", "index_var") or "index"
 624         return index_var
 625
 626     def get_nested_filename(self):
 627         """Returns the filename of the nested YAML file if the
 628         resource is a nested YAML or ResourceDef, returns '' otherwise."""
 629         typ = self.resource.get("type", "")
 630         if typ == "OS::Heat::ResourceGroup":
 631             rd = nested_dict.get(self.resource, "properties", "resource_def")
 632             typ = rd.get("type", "") if rd else ""
 633         ext = os.path.splitext(typ)[1]
 634         ext = ext.lower()
 635         if ext == ".yml" or ext == ".yaml":
 636             return typ
 637         else:
 638             return ""
 639
 640     def get_nested_properties(self):
 641         """
 642         Returns {} if not nested
 643         Returns resource: properties if nested
 644         Returns resource: properties: resource_def: properties if RG
 645         """
 646         if not bool(self.get_nested_filename()):
 647             return {}
 648         elif self.resource_type == "OS::Heat::ResourceGroup":
 649             return nested_dict.get(
 650                 self.properties, "resource_def", "properties", default={}
 651             )
 652         else:
 653             return self.properties
 654
 655     @property
 656     def depends_on(self):
 657         """
 658         Returns the list of resources this resource depends on.  Always
 659         returns a list.
 660
 661         :return: list of all resource IDs this resource depends on.  If none,
 662                  then returns an empty list
 663         """
 664         parents = self.resource.get("depends_on", [])
 665         return parents if isinstance(parents, list) else [parents]
 666
 667     def is_nested(self):
 668         """Returns True if the resource represents a Nested YAML resource
 669         using either type: {filename} or ResourceGroup -> resource_def"""
 670         return bool(self.get_nested_filename())
 671
 672     def get_nested_yaml(self, base_dir):
 673         """If the resource represents a Nested YAML resource, then it
 674         returns the loaded YAML.  If the resource is not nested or the
 675         file cannot be found, then an empty dict is returned"""
 676         filename = self.get_nested_filename()
 677         if filename:
 678             file_path = os.path.join(base_dir, filename)
 679             return load_yaml(file_path) if os.path.exists(file_path) else {}
 680         else:
 681             return {}
 682
 683
 684 def _get_heat_objects():
 685     """
 686     Introspect this module and return a dict of all HeatObject sub-classes with
 687     a (True) resource_type. Key is the resource_type, value is the
 688     corresponding class.
 689     """
 690     mod_classes = inspect.getmembers(sys.modules[__name__], inspect.isclass)
 691     heat_objects = {
 692         c.resource_type: c
 693         for _, c in mod_classes
 694         if issubclass(c, HeatObject) and c.resource_type
 695     }
 696     return heat_objects
 697
 698
 699 _HEAT_OBJECTS = _get_heat_objects()