2 # ============LICENSE_START====================================================
3 # org.onap.vvp/validation-scripts
4 # ===================================================================
5 # Copyright © 2019 AT&T Intellectual Property. All rights reserved.
6 # ===================================================================
8 # Unless otherwise specified, all software contained herein is licensed
9 # under the Apache License, Version 2.0 (the "License");
10 # you may not use this software except in compliance with the License.
11 # You may obtain a copy of the License at
13 # http://www.apache.org/licenses/LICENSE-2.0
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
23 # Unless otherwise specified, all documentation contained herein is licensed
24 # under the Creative Commons License, Attribution 4.0 Intl. (the "License");
25 # you may not use this documentation except in compliance with the License.
26 # You may obtain a copy of the License at
28 # https://creativecommons.org/licenses/by/4.0/
30 # Unless required by applicable law or agreed to in writing, documentation
31 # distributed under the License is distributed on an "AS IS" BASIS,
32 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
33 # See the License for the specific language governing permissions and
34 # limitations under the License.
36 # ============LICENSE_END============================================
39 from abc import ABC, abstractmethod
40 from collections import OrderedDict
42 from preload.generator import yield_by_count
43 from preload.environment import PreloadEnvironment
44 from tests.helpers import (
52 from tests.parametrizers import parametrize_heat_templates
53 from tests.structures import NeutronPortProcessor, Heat
54 from tests.test_environment_file_parameters import get_preload_excluded_parameters
55 from tests.utils import nested_dict
56 from tests.utils.vm_types import get_vm_type_for_nova_server
57 from config import Config, get_generator_plugins
62 # This is only used to fake out parametrizers
64 def __init__(self, config):
68 def parametrize(self, name, file_list):
69 self.inputs[name] = file_list
72 def get_heat_templates(config):
74 Returns the Heat template paths discovered by the pytest parameterizers
75 :param config: pytest config
76 :return: list of heat template paths
78 meta = DummyMetafunc(config)
79 parametrize_heat_templates(meta)
80 heat_templates = meta.inputs.get("heat_templates", [])
81 if isinstance(heat_templates, list) and len(heat_templates) > 0:
82 heat_templates = heat_templates[0]
88 class FilterBaseOutputs(ABC):
90 Invoked to remove parameters in an object that appear in the base module.
91 Base output parameters can be passed to incremental modules
92 so they do not need to be defined in a preload. This method can be
93 invoked on a module to pre-filter the parameters before a preload is
96 The method should remove the parameters that exist in the base module from
97 both itself and any sub-objects.
101 def filter_output_params(self, base_outputs):
102 raise NotImplementedError()
106 def __init__(self, ip_addr_param, port):
107 self.param = ip_addr_param or ""
111 def ip_version(self):
112 return 6 if "_v6_" in self.param else 4
115 return hash(self.param)
117 def __eq__(self, other):
118 return hash(self) == hash(other)
121 return "{}(v{})".format(self.param, self.ip_version)
127 class Network(FilterBaseOutputs):
128 def __init__(self, role, name_param):
129 self.network_role = role
130 self.name_param = name_param
131 self.subnet_params = set()
133 def filter_output_params(self, base_outputs):
134 self.subnet_params = remove(self.subnet_params, base_outputs)
137 return hash(self.network_role)
139 def __eq__(self, other):
140 return hash(self) == hash(other)
143 class Port(FilterBaseOutputs):
144 def __init__(self, vm, network):
146 self.network = network
148 self.floating_ips = []
149 self.uses_dhcp = True
151 def add_ips(self, props):
152 props = props.get("properties") or props
153 for fixed_ip in props.get("fixed_ips") or []:
154 if not isinstance(fixed_ip, dict):
156 ip_address = get_param(fixed_ip.get("ip_address"))
157 subnet = get_param(fixed_ip.get("subnet") or fixed_ip.get("subnet_id"))
159 self.uses_dhcp = False
160 self.fixed_ips.append(IpParam(ip_address, self))
162 self.network.subnet_params.add(subnet)
163 for ip in prop_iterator(props, "allowed_address_pairs", "ip_address"):
164 self.uses_dhcp = False
165 param = get_param(ip) if ip else ""
167 self.floating_ips.append(IpParam(param, self))
169 def filter_output_params(self, base_outputs):
170 self.fixed_ips = remove(self.fixed_ips, base_outputs, key=lambda ip: ip.param)
171 self.floating_ips = remove(
172 self.floating_ips, base_outputs, key=lambda ip: ip.param
176 class VirtualMachineType(FilterBaseOutputs):
177 def __init__(self, vm_type, vnf_module):
178 self.vm_type = vm_type
182 self.vnf_module = vnf_module
184 def filter_output_params(self, base_outputs):
185 self.names = remove(self.names, base_outputs)
186 for port in self.ports:
187 port.filter_output_params(base_outputs)
191 return {port.network for port in self.ports}
194 def floating_ips(self):
195 for port in self.ports:
196 for ip in port.floating_ips:
201 for port in self.ports:
202 for ip in port.fixed_ips:
205 def update_ports(self, network, props):
206 port = self.get_or_create_port(network)
209 def get_or_create_port(self, network):
210 for port in self.ports:
211 if port.network == network:
213 port = Port(self, network)
214 self.ports.append(port)
219 def __init__(self, templates):
220 self.modules = [VnfModule(t, self) for t in templates]
221 self.uses_contrail = self._uses_contrail()
222 self.base_module = next(
223 (mod for mod in self.modules if mod.is_base_module), None
225 self.incremental_modules = [m for m in self.modules if not m.is_base_module]
227 def _uses_contrail(self):
228 for mod in self.modules:
229 resources = mod.heat.get_all_resources()
230 types = (r.get("type", "") for r in resources.values())
231 if any(t.startswith("OS::ContrailV2") for t in types):
236 def base_output_params(self):
237 return self.base_module.heat.outputs
239 def filter_base_outputs(self):
240 non_base_modules = (m for m in self.modules if not m.is_base_module)
241 for mod in non_base_modules:
242 mod.filter_output_params(self.base_output_params)
245 def env_path(heat_path):
247 Create the path to the env file for the give heat path.
248 :param heat_path: path to heat file
249 :return: path to env file (assumes it is present and named correctly)
251 base_path = os.path.splitext(heat_path)[0]
252 return "{}.env".format(base_path)
255 class VnfModule(FilterBaseOutputs):
256 def __init__(self, template_file, vnf):
258 self.vnf_name = os.path.splitext(os.path.basename(template_file))[0]
259 self.template_file = template_file
260 self.heat = Heat(filepath=template_file, envpath=env_path(template_file))
261 env_pair = get_environment_pair(self.template_file)
262 env_yaml = env_pair.get("eyml") if env_pair else {}
263 self.parameters = env_yaml.get("parameters") or {}
265 self.virtual_machine_types = self._create_vm_types()
267 self.outputs_filtered = False
269 def filter_output_params(self, base_outputs):
270 for vm in self.virtual_machine_types:
271 vm.filter_output_params(base_outputs)
272 for network in self.networks:
273 network.filter_output_params(base_outputs)
275 k: v for k, v in self.parameters.items() if k not in base_outputs
279 for network in self.networks
280 if network.name_param not in base_outputs or network.subnet_params
282 self.outputs_filtered = True
284 def _create_vm_types(self):
285 servers = self.heat.get_resource_by_type("OS::Nova::Server", all_resources=True)
287 for _, props in yield_by_count(servers):
288 vm_type = get_vm_type_for_nova_server(props)
289 vm = vm_types.setdefault(vm_type, VirtualMachineType(vm_type, self))
291 name = nested_dict.get(props, "properties", "name", default={})
292 vm_name = get_param(name) if name else ""
293 vm.names.append(vm_name)
294 return list(vm_types.values())
296 def _add_networks(self):
297 ports = self.heat.get_resource_by_type("OS::Neutron::Port", all_resources=True)
298 for rid, props in yield_by_count(ports):
299 resource_type, port_match = NeutronPortProcessor.get_rid_match_tuple(rid)
300 if resource_type != "external":
302 network_role = port_match.group("network_role")
303 vm = self._get_vm_type(port_match.group("vm_type"))
304 network = self._get_network(network_role, props)
305 vm.update_ports(network, props)
308 def is_base_module(self):
309 return is_base_module(self.template_file)
312 def availability_zones(self):
313 """Returns a list of all availability zone parameters found in the template"""
315 p for p in self.heat.parameters if p.startswith("availability_zone")
321 Label for the VF module that will appear in the CSAR
327 """Return available Environment Spec definitions"""
328 return Config().env_specs
331 def env_template(self):
333 Returns a a template .env file that can be completed to enable
336 params = OrderedDict()
337 params["vnf_name"] = CHANGE
338 params["vnf-type"] = CHANGE
339 params["vf-module-model-name"] = CHANGE
340 params["vf_module_name"] = CHANGE
341 for az in self.availability_zones:
343 for network in self.networks:
344 params[network.name_param] = CHANGE
345 for param in set(network.subnet_params):
346 params[param] = CHANGE
347 for vm in self.virtual_machine_types:
348 for name in set(vm.names):
349 params[name] = CHANGE
350 for ip in vm.floating_ips:
351 params[ip.param] = CHANGE
352 for ip in vm.fixed_ips:
353 params[ip.param] = CHANGE
354 excluded = get_preload_excluded_parameters(
355 self.template_file, persistent_only=True
357 for name, value in self.parameters.items():
361 return {"parameters": params}
364 def preload_parameters(self):
366 Subset of parameters from the env file that can be overridden in
367 tag values. Per VNF Heat Guidelines, specific parameters such as
368 flavor, image, etc. must not be overridden so they are excluded.
370 :return: dict of parameters suitable for the preload
372 excluded = get_preload_excluded_parameters(self.template_file)
373 return {k: v for k, v in self.parameters.items() if k not in excluded}
375 def _get_vm_type(self, vm_type):
376 for vm in self.virtual_machine_types:
377 if vm_type.lower() == vm.vm_type.lower():
379 raise RuntimeError("Encountered unknown VM type: {}".format(vm_type))
381 def _get_network(self, network_role, props):
382 network_prop = nested_dict.get(props, "properties", "network") or {}
383 name_param = get_param(network_prop) if network_prop else ""
384 for network in self.networks:
385 if network.network_role.lower() == network_role.lower():
387 new_network = Network(network_role, name_param)
388 self.networks.append(new_network)
392 return "VNF Module ({})".format(os.path.basename(self.template_file))
398 return hash(self.vnf_name)
400 def __eq__(self, other):
401 return hash(self) == hash(other)
404 def create_preloads(config, exitstatus):
406 Create preloads in every format that can be discovered by get_generator_plugins
408 if config.getoption("self_test"):
410 print("+===================================================================+")
411 print("| Preload Template Generation |")
412 print("+===================================================================+")
414 preload_dir = os.path.join(get_output_dir(config), "preloads")
415 if os.path.exists(preload_dir):
416 shutil.rmtree(preload_dir)
417 env_directory = config.getoption("env_dir")
418 preload_env = PreloadEnvironment(env_directory) if env_directory else None
419 plugins = get_generator_plugins()
420 available_formats = [p.format_name() for p in plugins]
421 selected_formats = config.getoption("preload_formats") or available_formats
422 heat_templates = get_heat_templates(config)
424 for plugin_class in plugins:
425 if plugin_class.format_name() not in selected_formats:
427 vnf = Vnf(heat_templates)
428 generator = plugin_class(vnf, preload_dir, preload_env)
430 if vnf and vnf.uses_contrail:
432 "\nWARNING: Preload template generation does not support Contrail\n"
433 "at this time, but Contrail resources were detected. The preload \n"
434 "template may be incomplete."
438 "\nWARNING: Heat violations detected. Preload templates may be\n"