2 # ============LICENSE_START====================================================
3 # org.onap.vvp/validation-scripts
4 # ===================================================================
5 # Copyright © 2019 AT&T Intellectual Property. All rights reserved.
6 # ===================================================================
8 # Unless otherwise specified, all software contained herein is licensed
9 # under the Apache License, Version 2.0 (the "License");
10 # you may not use this software except in compliance with the License.
11 # You may obtain a copy of the License at
13 # http://www.apache.org/licenses/LICENSE-2.0
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
23 # Unless otherwise specified, all documentation contained herein is licensed
24 # under the Creative Commons License, Attribution 4.0 Intl. (the "License");
25 # you may not use this documentation except in compliance with the License.
26 # You may obtain a copy of the License at
28 # https://creativecommons.org/licenses/by/4.0/
30 # Unless required by applicable law or agreed to in writing, documentation
31 # distributed under the License is distributed on an "AS IS" BASIS,
32 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
33 # See the License for the specific language governing permissions and
34 # limitations under the License.
36 # ============LICENSE_END============================================
39 from abc import ABC, abstractmethod
41 from preload.generator import yield_by_count
42 from preload.environment import PreloadEnvironment
43 from tests.helpers import (
51 from tests.parametrizers import parametrize_heat_templates
52 from tests.structures import NeutronPortProcessor, Heat
53 from tests.test_environment_file_parameters import get_preload_excluded_parameters
54 from tests.utils import nested_dict
55 from tests.utils.vm_types import get_vm_type_for_nova_server
56 from config import Config, get_generator_plugins
61 # This is only used to fake out parametrizers
63 def __init__(self, config):
67 def parametrize(self, name, file_list):
68 self.inputs[name] = file_list
71 def get_heat_templates(config):
73 Returns the Heat template paths discovered by the pytest parameterizers
74 :param config: pytest config
75 :return: list of heat template paths
77 meta = DummyMetafunc(config)
78 parametrize_heat_templates(meta)
79 heat_templates = meta.inputs.get("heat_templates", [])
80 if isinstance(heat_templates, list) and len(heat_templates) > 0:
81 heat_templates = heat_templates[0]
87 class FilterBaseOutputs(ABC):
89 Invoked to remove parameters in an object that appear in the base module.
90 Base output parameters can be passed to incremental modules
91 so they do not need to be defined in a preload. This method can be
92 invoked on a module to pre-filter the parameters before a preload is
95 The method should remove the parameters that exist in the base module from
96 both itself and any sub-objects.
100 def filter_output_params(self, base_outputs):
101 raise NotImplementedError()
105 def __init__(self, ip_addr_param, port):
106 self.param = ip_addr_param or ""
110 def ip_version(self):
111 return 6 if "_v6_" in self.param else 4
114 return hash(self.param)
116 def __eq__(self, other):
117 return hash(self) == hash(other)
120 return "{}(v{})".format(self.param, self.ip_version)
126 class Network(FilterBaseOutputs):
127 def __init__(self, role, name_param):
128 self.network_role = role
129 self.name_param = name_param
130 self.subnet_params = set()
132 def filter_output_params(self, base_outputs):
133 self.subnet_params = remove(self.subnet_params, base_outputs)
136 return hash(self.network_role)
138 def __eq__(self, other):
139 return hash(self) == hash(other)
142 class Port(FilterBaseOutputs):
143 def __init__(self, vm, network):
145 self.network = network
147 self.floating_ips = []
148 self.uses_dhcp = True
150 def add_ips(self, props):
151 props = props.get("properties") or props
152 for fixed_ip in props.get("fixed_ips") or []:
153 if not isinstance(fixed_ip, dict):
155 ip_address = get_param(fixed_ip.get("ip_address"))
156 subnet = get_param(fixed_ip.get("subnet") or fixed_ip.get("subnet_id"))
158 self.uses_dhcp = False
159 self.fixed_ips.append(IpParam(ip_address, self))
161 self.network.subnet_params.add(subnet)
162 for ip in prop_iterator(props, "allowed_address_pairs", "ip_address"):
163 self.uses_dhcp = False
164 param = get_param(ip) if ip else ""
166 self.floating_ips.append(IpParam(param, self))
168 def filter_output_params(self, base_outputs):
169 self.fixed_ips = remove(self.fixed_ips, base_outputs, key=lambda ip: ip.param)
170 self.floating_ips = remove(
171 self.floating_ips, base_outputs, key=lambda ip: ip.param
175 class VirtualMachineType(FilterBaseOutputs):
176 def __init__(self, vm_type, vnf_module):
177 self.vm_type = vm_type
181 self.vnf_module = vnf_module
183 def filter_output_params(self, base_outputs):
184 self.names = remove(self.names, base_outputs)
185 for port in self.ports:
186 port.filter_output_params(base_outputs)
190 return {port.network for port in self.ports}
193 def floating_ips(self):
194 for port in self.ports:
195 for ip in port.floating_ips:
200 for port in self.ports:
201 for ip in port.fixed_ips:
204 def update_ports(self, network, props):
205 port = self.get_or_create_port(network)
208 def get_or_create_port(self, network):
209 for port in self.ports:
210 if port.network == network:
212 port = Port(self, network)
213 self.ports.append(port)
218 def __init__(self, templates):
219 self.modules = [VnfModule(t, self) for t in templates]
220 self.uses_contrail = self._uses_contrail()
221 self.base_module = next(
222 (mod for mod in self.modules if mod.is_base_module), None
224 self.incremental_modules = [m for m in self.modules if not m.is_base_module]
226 def _uses_contrail(self):
227 for mod in self.modules:
228 resources = mod.heat.get_all_resources()
229 types = (r.get("type", "") for r in resources.values())
230 if any(t.startswith("OS::ContrailV2") for t in types):
235 def base_output_params(self):
236 return self.base_module.heat.outputs
238 def filter_base_outputs(self):
239 non_base_modules = (m for m in self.modules if not m.is_base_module)
240 for mod in non_base_modules:
241 mod.filter_output_params(self.base_output_params)
244 def env_path(heat_path):
246 Create the path to the env file for the give heat path.
247 :param heat_path: path to heat file
248 :return: path to env file (assumes it is present and named correctly)
250 base_path = os.path.splitext(heat_path)[0]
251 return "{}.env".format(base_path)
254 class VnfModule(FilterBaseOutputs):
255 def __init__(self, template_file, vnf):
257 self.vnf_name = os.path.splitext(os.path.basename(template_file))[0]
258 self.template_file = template_file
259 self.heat = Heat(filepath=template_file, envpath=env_path(template_file))
260 env_pair = get_environment_pair(self.template_file)
261 env_yaml = env_pair.get("eyml") if env_pair else {}
262 self.parameters = env_yaml.get("parameters") or {}
264 self.virtual_machine_types = self._create_vm_types()
266 self.outputs_filtered = False
268 def filter_output_params(self, base_outputs):
269 for vm in self.virtual_machine_types:
270 vm.filter_output_params(base_outputs)
271 for network in self.networks:
272 network.filter_output_params(base_outputs)
274 k: v for k, v in self.parameters.items() if k not in base_outputs
278 for network in self.networks
279 if network.name_param not in base_outputs or network.subnet_params
281 self.outputs_filtered = True
283 def _create_vm_types(self):
284 servers = self.heat.get_resource_by_type("OS::Nova::Server", all_resources=True)
286 for _, props in yield_by_count(servers):
287 vm_type = get_vm_type_for_nova_server(props)
288 vm = vm_types.setdefault(vm_type, VirtualMachineType(vm_type, self))
290 name = nested_dict.get(props, "properties", "name", default={})
291 vm_name = get_param(name) if name else ""
292 vm.names.append(vm_name)
293 return list(vm_types.values())
295 def _add_networks(self):
296 ports = self.heat.get_resource_by_type("OS::Neutron::Port", all_resources=True)
297 for rid, props in yield_by_count(ports):
298 resource_type, port_match = NeutronPortProcessor.get_rid_match_tuple(rid)
299 if resource_type != "external":
301 network_role = port_match.group("network_role")
302 vm = self._get_vm_type(port_match.group("vm_type"))
303 network = self._get_network(network_role, props)
304 vm.update_ports(network, props)
307 def is_base_module(self):
308 return is_base_module(self.template_file)
311 def availability_zones(self):
312 """Returns a list of all availability zone parameters found in the template"""
314 p for p in self.heat.parameters if p.startswith("availability_zone")
320 Label for the VF module that will appear in the CSAR
326 """Return available Environment Spec definitions"""
327 return Config().env_specs
330 def env_template(self):
332 Returns a a template .env file that can be completed to enable
336 params["vnf-name"] = CHANGE
337 params["vnf-type"] = CHANGE
338 params["vf-module-model-name"] = CHANGE
339 params["vf_module_name"] = CHANGE
340 for network in self.networks:
341 params[network.name_param] = CHANGE
342 for param in set(network.subnet_params):
343 params[param] = CHANGE
344 for vm in self.virtual_machine_types:
345 for name in set(vm.names):
346 params[name] = CHANGE
347 for ip in vm.floating_ips:
348 params[ip.param] = CHANGE
349 for ip in vm.fixed_ips:
350 params[ip.param] = CHANGE
351 excluded = get_preload_excluded_parameters(
352 self.template_file, persistent_only=True
354 for name, value in self.parameters.items():
358 return {"parameters": params}
361 def preload_parameters(self):
363 Subset of parameters from the env file that can be overridden in
364 tag values. Per VNF Heat Guidelines, specific parameters such as
365 flavor, image, etc. must not be overridden so they are excluded.
367 :return: dict of parameters suitable for the preload
369 excluded = get_preload_excluded_parameters(self.template_file)
370 return {k: v for k, v in self.parameters.items() if k not in excluded}
372 def _get_vm_type(self, vm_type):
373 for vm in self.virtual_machine_types:
374 if vm_type.lower() == vm.vm_type.lower():
376 raise RuntimeError("Encountered unknown VM type: {}".format(vm_type))
378 def _get_network(self, network_role, props):
379 network_prop = nested_dict.get(props, "properties", "network") or {}
380 name_param = get_param(network_prop) if network_prop else ""
381 for network in self.networks:
382 if network.network_role.lower() == network_role.lower():
384 new_network = Network(network_role, name_param)
385 self.networks.append(new_network)
389 return "VNF Module ({})".format(os.path.basename(self.template_file))
395 return hash(self.vnf_name)
397 def __eq__(self, other):
398 return hash(self) == hash(other)
401 def create_preloads(config, exitstatus):
403 Create preloads in every format that can be discovered by get_generator_plugins
405 if config.getoption("self_test"):
407 print("+===================================================================+")
408 print("| Preload Template Generation |")
409 print("+===================================================================+")
411 preload_dir = os.path.join(get_output_dir(config), "preloads")
412 if os.path.exists(preload_dir):
413 shutil.rmtree(preload_dir)
414 env_directory = config.getoption("env_dir")
415 preload_env = PreloadEnvironment(env_directory) if env_directory else None
416 plugins = get_generator_plugins()
417 available_formats = [p.format_name() for p in plugins]
418 selected_formats = config.getoption("preload_formats") or available_formats
419 heat_templates = get_heat_templates(config)
421 for plugin_class in plugins:
422 if plugin_class.format_name() not in selected_formats:
424 vnf = Vnf(heat_templates)
425 generator = plugin_class(vnf, preload_dir, preload_env)
427 if vnf and vnf.uses_contrail:
429 "\nWARNING: Preload template generation does not support Contrail\n"
430 "at this time, but Contrail resources were detected. The preload \n"
431 "template may be incomplete."
435 "\nWARNING: Heat violations detected. Preload templates may be\n"