osdf/adapters/conductor/translation.py

   1 # -------------------------------------------------------------------------
   2 #   Copyright (c) 2015-2017 AT&T Intellectual Property
   3 #   Copyright (C) 2020 Wipro Limited.
   4 #
   5 #   Licensed under the Apache License, Version 2.0 (the "License");
   6 #   you may not use this file except in compliance with the License.
   7 #   You may obtain a copy of the License at
   8 #
   9 #       http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 #   Unless required by applicable law or agreed to in writing, software
  12 #   distributed under the License is distributed on an "AS IS" BASIS,
  13 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 #   See the License for the specific language governing permissions and
  15 #   limitations under the License.
  16 #
  17 # -------------------------------------------------------------------------
  18 #
  19 import copy
  20 import json
  21 import re
  22 import yaml
  23
  24 from osdf.utils.programming_utils import dot_notation
  25
  26 policy_config_mapping = yaml.safe_load(open('config/has_config.yaml')).get('policy_config_mapping')
  27
  28 CONSTRAINT_TYPE_MAP = {"onap.policies.optimization.resource.AttributePolicy": "attribute",
  29                        "onap.policies.optimization.resource.DistancePolicy": "distance_to_location",
  30                        "onap.policies.optimization.resource.InventoryGroupPolicy": "inventory_group",
  31                        "onap.policies.optimization.resource.ResourceInstancePolicy": "instance_fit",
  32                        "onap.policies.optimization.resource.ResourceRegionPolicy": "region_fit",
  33                        "onap.policies.optimization.resource.AffinityPolicy": "zone",
  34                        "onap.policies.optimization.resource.InstanceReservationPolicy":
  35                            "instance_reservation",
  36                        "onap.policies.optimization.resource.Vim_fit": "vim_fit",
  37                        "onap.policies.optimization.resource.HpaPolicy": "hpa",
  38                        "onap.policies.optimization.resource.ThresholdPolicy": "threshold",
  39                        "onap.policies.optimization.resource.AggregationPolicy": "aggregation"
  40                        }
  41
  42
  43 def get_opt_query_data(request_parameters, policies):
  44     """Fetch service and order specific details from the requestParameters field of a request.
  45
  46         :param request_parameters: A list of request parameters
  47         :param policies: A set of policies
  48         :return: A dictionary with service and order-specific attributes.
  49         """
  50     req_param_dict = {}
  51     if request_parameters:
  52         for policy in policies:
  53             for queryProp in policy[list(policy.keys())[0]]['properties']['queryProperties']:
  54                 attr_val = queryProp['value'] if 'value' in queryProp and queryProp['value'] != "" \
  55                     else dot_notation(request_parameters, queryProp['attribute_location'])
  56                 if attr_val is not None:
  57                     req_param_dict.update({queryProp['attribute']: attr_val})
  58     return req_param_dict
  59
  60
  61 def gen_optimization_policy(vnf_list, optimization_policy):
  62     """Generate optimization policy details to pass to Conductor
  63
  64     :param vnf_list: List of vnf's to used in placement request
  65     :param optimization_policy: optimization objective policy information provided in the incoming request
  66     :return: List of optimization objective policies in a format required by Conductor
  67     """
  68     if len(optimization_policy) == 1:
  69         policy = optimization_policy[0]
  70         policy_content = policy[list(policy.keys())[0]]
  71         if policy_content['type_version'] == '2.0.0':
  72             properties = policy_content['properties']
  73             objective = {'goal': properties['goal'],
  74                          'operation_function': properties['operation_function']}
  75             return [objective]
  76
  77     optimization_policy_list = []
  78     for policy in optimization_policy:
  79         content = policy[list(policy.keys())[0]]['properties']
  80         parameter_list = []
  81         parameters = ["cloud_version", "hpa_score"]
  82
  83         for attr in content['objectiveParameter']['parameterAttributes']:
  84             parameter = attr['parameter'] if attr['parameter'] in parameters else attr['parameter'] + "_between"
  85             default, vnfs = get_matching_vnfs(attr['resources'], vnf_list)
  86             for vnf in vnfs:
  87                 value = [vnf] if attr['parameter'] in parameters else [attr['customerLocationInfo'], vnf]
  88                 parameter_list.append({
  89                     attr['operator']: [attr['weight'], {parameter: value}]
  90                 })
  91
  92         optimization_policy_list.append({
  93             content['objective']: {content['objectiveParameter']['operator']: parameter_list}
  94         })
  95     return optimization_policy_list
  96
  97
  98 def get_matching_vnfs(resources, vnf_list, match_type="intersection"):
  99     """Get a list of matching VNFs from the list of resources
 100
 101     :param resources:
 102     :param vnf_list: List of vnfs to used in placement request
 103     :param match_type: "intersection" or "all" or "any" (any => send all_vnfs if there is any intersection)
 104     :return: List of matching VNFs
 105     """
 106     # Check if it is a default policy
 107     default = True if resources == [] else False
 108     resources_lcase = [x.lower() for x in resources] if not default else [x.lower() for x in vnf_list]
 109     if match_type == "all":  # don't bother with any comparisons
 110         return default, resources if set(resources_lcase) <= set(vnf_list) else None
 111     common_vnfs = set(vnf_list) & set(resources_lcase) if not default else set(vnf_list)
 112     common_resources = [x for x in resources if x.lower() in common_vnfs] if not default else list(common_vnfs)
 113     if match_type == "intersection":  # specifically requested intersection
 114         return default, list(common_resources)
 115     return default, resources if common_vnfs else None  # "any" match => all resources to be returned
 116
 117
 118 def gen_policy_instance(vnf_list, resource_policy, match_type="intersection", rtype=None):
 119     """Generate a list of policies
 120
 121     :param vnf_list: List of vnf's to used in placement request
 122     :param resource_policy: policy for this specific resource
 123     :param match_type: How to match the vnf_names with the vnf_list (intersection or "any")
 124              intersection => return intersection; "any" implies return all vnf_names if intersection is not null
 125     :param rtype: resource type (e.g. resourceRegionProperty or resourceInstanceProperty)
 126              None => no controller information added to the policy specification to Conductor
 127     :return: resource policy list in a format required by Conductor
 128     """
 129     resource_policy_list = []
 130     related_policies = []
 131     for policy in resource_policy:
 132         pc = policy[list(policy.keys())[0]]
 133         default, demands = get_matching_vnfs(pc['properties']['resources'], vnf_list, match_type=match_type)
 134         resource = {pc['properties']['identity']: {'type': CONSTRAINT_TYPE_MAP.get(pc['type']), 'demands': demands}}
 135
 136         if rtype:
 137             resource[pc['properties']['identity']]['properties'] = {'controller': pc[rtype]['controller'],
 138                                                                     'request': json.loads(pc[rtype]['request'])}
 139         if demands and len(demands) != 0:
 140             # The default policy shall not override the specific policy that already appended
 141             if default:
 142                 for d in demands:
 143                     resource_repeated = True \
 144                         if {pc['properties']['identity']: {'type': CONSTRAINT_TYPE_MAP.get(pc['type']),
 145                                                            'demands': d}} in resource_policy_list else False
 146                     if resource_repeated:
 147                         continue
 148                     else:
 149                         resource_policy_list.append(
 150                             {pc['properties']['identity']: {'type': CONSTRAINT_TYPE_MAP.get(pc['type']),
 151                                                             'demands': d}})
 152                         policy[list(policy.keys())[0]]['properties']['resources'] = d
 153                         related_policies.append(policy)
 154             # Need to override the default policies, here delete the outdated policy stored in the db
 155             if resource in resource_policy_list:
 156                 for pc in related_policies:
 157                     if pc[list(pc.keys()[0])]['properties']['resources'] == resource:
 158                         related_policies.remove(pc)
 159                 resource_policy_list.remove(resource)
 160             related_policies.append(policy)
 161             resource_policy_list.append(resource)
 162
 163     return resource_policy_list, related_policies
 164
 165
 166 def gen_resource_instance_policy(vnf_list, resource_instance_policy):
 167     """Get policies governing resource instances in order to populate the Conductor API call"""
 168     cur_policies, _ = gen_policy_instance(vnf_list, resource_instance_policy, rtype='resourceInstanceProperty')
 169     return cur_policies
 170
 171
 172 def gen_resource_region_policy(vnf_list, resource_region_policy):
 173     """Get policies governing resource region in order to populate the Conductor API call"""
 174     cur_policies, _ = gen_policy_instance(vnf_list, resource_region_policy, rtype='resourceRegionProperty')
 175     return cur_policies
 176
 177
 178 def gen_inventory_group_policy(vnf_list, inventory_group_policy):
 179     """Get policies governing inventory group in order to populate the Conductor API call"""
 180     cur_policies, _ = gen_policy_instance(vnf_list, inventory_group_policy, rtype=None)
 181     return cur_policies
 182
 183
 184 def gen_reservation_policy(vnf_list, reservation_policy):
 185     """Get policies governing resource instances in order to populate the Conductor API call"""
 186     cur_policies, _ = gen_policy_instance(vnf_list, reservation_policy, rtype='instanceReservationProperty')
 187     return cur_policies
 188
 189
 190 def gen_distance_to_location_policy(vnf_list, distance_to_location_policy):
 191     """Get policies governing distance-to-location for VNFs in order to populate the Conductor API call"""
 192     cur_policies, related_policies = gen_policy_instance(vnf_list, distance_to_location_policy, rtype=None)
 193     for p_new, p_main in zip(cur_policies, related_policies):  # add additional fields to each policy
 194         properties = p_main[list(p_main.keys())[0]]['properties']['distanceProperties']
 195         pcp_d = properties['distance']
 196         p_new[p_main[list(p_main.keys())[0]]['properties']['identity']]['properties'] = {
 197             'distance': pcp_d['operator'] + " " + pcp_d['value'].lower() + " " + pcp_d['unit'].lower(),
 198             'location': properties['locationInfo']
 199         }
 200     return cur_policies
 201
 202
 203 def gen_attribute_policy(vnf_list, attribute_policy):
 204     """Get policies governing attributes of VNFs in order to populate the Conductor API call"""
 205     cur_policies, related_policies = gen_policy_instance(vnf_list, attribute_policy, rtype=None)
 206     for p_new, p_main in zip(cur_policies, related_policies):  # add additional fields to each policy
 207         properties = p_main[list(p_main.keys())[0]]['properties']['attributeProperties']
 208         attribute_mapping = policy_config_mapping['filtering_attributes']  # wanted attributes and mapping
 209         p_new[p_main[list(p_main.keys())[0]]['properties']['identity']]['properties'] = {
 210             'evaluate': dict((attribute_mapping[k], properties.get(k)
 211                               if k != "cloudRegion" else gen_cloud_region(properties))
 212                              for k in attribute_mapping.keys())
 213         }
 214     return cur_policies  # cur_policies gets updated in place...
 215
 216
 217 def gen_zone_policy(vnf_list, zone_policy):
 218     """Get zone policies in order to populate the Conductor API call"""
 219     cur_policies, related_policies = gen_policy_instance(vnf_list, zone_policy, match_type="all", rtype=None)
 220     for p_new, p_main in zip(cur_policies, related_policies):  # add additional fields to each policy
 221         pmz = p_main[list(p_main.keys())[0]]['properties']['affinityProperties']
 222         p_new[p_main[list(p_main.keys())[0]]['properties']['identity']]['properties'] = \
 223             {'category': pmz['category'], 'qualifier': pmz['qualifier']}
 224     return cur_policies
 225
 226
 227 def gen_capacity_policy(vnf_list, capacity_policy):
 228     """Get zone policies in order to populate the Conductor API call"""
 229     cur_policies, related_policies = gen_policy_instance(vnf_list, capacity_policy, rtype=None)
 230     for p_new, p_main in zip(cur_policies, related_policies):  # add additional fields to each policy
 231         pmz = p_main[list(p_main.keys())[0]]['properties']['capacityProperty']
 232         p_new[p_main[list(p_main.keys())[0]]['properties']['identity']]['properties'] = \
 233             {"controller": pmz['controller'], 'request': json.loads(pmz['request'])}
 234     return cur_policies
 235
 236
 237 def gen_hpa_policy(vnf_list, hpa_policy):
 238     """Get zone policies in order to populate the Conductor API call"""
 239     cur_policies, related_policies = gen_policy_instance(vnf_list, hpa_policy, rtype=None)
 240     for p_new, p_main in zip(cur_policies, related_policies):  # add additional fields to each policy
 241         p_new[p_main[list(p_main.keys())[0]]['properties']['identity']]['properties'] = \
 242             {'evaluate': p_main[list(p_main.keys())[0]]['properties']['flavorFeatures']}
 243     return cur_policies
 244
 245
 246 def gen_threshold_policy(vnf_list, threshold_policy):
 247     cur_policies, related_policies = gen_policy_instance(vnf_list, threshold_policy, rtype=None)
 248     for p_new, p_main in zip(cur_policies, related_policies):
 249         pmz = p_main[list(p_main.keys())[0]]['properties']['thresholdProperties']
 250         p_new[p_main[list(p_main.keys())[0]]['properties']['identity']]['properties'] = {'evaluate': pmz}
 251     return cur_policies
 252
 253
 254 def gen_aggregation_policy(vnf_list, cross_policy):
 255     cur_policies, related_policies = gen_policy_instance(vnf_list, cross_policy, rtype=None)
 256     for p_new, p_main in zip(cur_policies, related_policies):
 257         pmz = p_main[list(p_main.keys())[0]]['properties']['aggregationProperties']
 258         p_new[p_main[list(p_main.keys())[0]]['properties']['identity']]['properties'] = {'evaluate': pmz}
 259     return cur_policies
 260
 261
 262 def get_augmented_policy_attributes(policy_property, demand):
 263     """Get policy attributes and augment them using policy_config_mapping and demand information"""
 264     attributes = copy.copy(policy_property['attributes'])
 265     remapping = policy_config_mapping['remapping']
 266     extra = dict((x, demand['resourceModelInfo'][remapping[x]]) for x in attributes if x in remapping)
 267     attributes.update(extra)
 268     return attributes
 269
 270
 271 def get_candidates_demands(demand):
 272     """Get demands related to candidates; e.g. excluded/required"""
 273     res = {}
 274     for k, v in policy_config_mapping['candidates'].items():
 275         if k not in demand:
 276             continue
 277         res[v] = [{'inventory_type': x['identifierType'], 'candidate_id': x['identifiers']} for x in demand[k]]
 278     return res
 279
 280
 281 def get_policy_properties(demand, policies):
 282     """Get policy_properties for cases where there is a match with the demand"""
 283     for policy in policies:
 284         policy_demands = set([x.lower() for x in policy[list(policy.keys())[0]]['properties']['resources']])
 285         if policy_demands and demand['resourceModuleName'].lower() not in policy_demands:
 286             continue  # no match for this policy
 287         elif policy_demands == set():  # Append resource name for default policy
 288             policy[list(policy.keys())[0]]['properties'].update(resources=list(demand.get('resourceModuleName')))
 289         for policy_property in policy[list(policy.keys())[0]]['properties']['vnfProperties']:
 290             yield policy_property
 291
 292
 293 def get_demand_properties(demand, policies):
 294     """Get list demand properties objects (named tuples) from policy"""
 295     demand_properties = []
 296     for policy_property in get_policy_properties(demand, policies):
 297         prop = dict(inventory_provider=policy_property['inventoryProvider'],
 298                     inventory_type=policy_property['inventoryType'],
 299                     service_type=demand.get('serviceResourceId', ''),
 300                     service_resource_id=demand.get('serviceResourceId', ''))
 301         policy_property_mapping = {'filtering_attributes': 'attributes',
 302                                    'passthrough_attributes': 'passthroughAttributes',
 303                                    'default_attributes': 'defaultAttributes'}
 304
 305         prop.update({'unique': policy_property['unique']} if 'unique' in policy_property and
 306                                                              policy_property['unique'] else {})
 307         prop['filtering_attributes'] = dict()
 308         for key, value in policy_property_mapping.items():
 309             get_demand_attributes(prop, policy_property, key, value)
 310
 311         prop['filtering_attributes'].update({'global-customer-id': policy_property['customerId']}
 312                                             if 'customerId' in policy_property and policy_property['customerId']
 313                                             else {})
 314         prop['filtering_attributes'].update({'model-invariant-id': demand['resourceModelInfo']['modelInvariantId']}
 315                                             if 'modelInvariantId' in demand['resourceModelInfo']
 316                                                and demand['resourceModelInfo']['modelInvariantId'] else {})
 317         prop['filtering_attributes'].update({'model-version-id': demand['resourceModelInfo']['modelVersionId']}
 318                                             if 'modelVersionId' in demand['resourceModelInfo']
 319                                                and demand['resourceModelInfo']['modelVersionId'] else {})
 320         prop['filtering_attributes'].update({'equipment-role': policy_property['equipmentRole']}
 321                                             if 'equipmentRole' in policy_property and policy_property['equipmentRole']
 322                                             else {})
 323
 324         prop.update(get_candidates_demands(demand))
 325         demand_properties.append(prop)
 326     return demand_properties
 327
 328
 329 def get_demand_attributes(prop, policy_property, attribute_type, key):
 330     if policy_property.get(key):
 331         prop[attribute_type] = dict()
 332         for attr_key, attr_val in policy_property[key].items():
 333             update_converted_attribute(attr_key, attr_val, prop, attribute_type)
 334
 335
 336 def update_converted_attribute(attr_key, attr_val, properties, attribute_type):
 337     """Updates dictonary of attributes with one specified in the arguments.
 338
 339     Automatically translates key namr from camelCase to hyphens
 340     :param attribute_type: attribute section name
 341     :param attr_key: key of the attribute
 342     :param attr_val: value of the attribute
 343     :param properties: dictionary with attributes to update
 344     :return:
 345     """
 346     if attr_val:
 347         remapping = policy_config_mapping[attribute_type]
 348         if remapping.get(attr_key):
 349             key_value = remapping.get(attr_key)
 350         else:
 351             key_value = re.sub('(.)([A-Z][a-z]+)', r'\1-\2', attr_key)
 352             key_value = re.sub('([a-z0-9])([A-Z])', r'\1-\2', key_value).lower()
 353         properties[attribute_type].update({key_value: attr_val})
 354
 355
 356 def gen_demands(demands, vnf_policies):
 357     """Generate list of demands based on request and VNF policies
 358
 359     :param demands: A List of demands
 360     :param vnf_policies: Policies associated with demand resources
 361            (e.g. from grouped_policies['vnfPolicy'])
 362     :return: list of demand parameters to populate the Conductor API call
 363     """
 364     demand_dictionary = {}
 365     for demand in demands:
 366         prop = get_demand_properties(demand, vnf_policies)
 367         if len(prop) > 0:
 368             demand_dictionary.update({demand['resourceModuleName']: prop})
 369     return demand_dictionary
 370
 371
 372 def gen_cloud_region(property):
 373     prop = {"cloud_region_attributes": dict()}
 374     if 'cloudRegion' in property:
 375         for k, v in property['cloudRegion'].items():
 376             update_converted_attribute(k, v, prop, 'cloud_region_attributes')
 377     return prop["cloud_region_attributes"]