X-Git-Url: https://gerrit.onap.org/r/gitweb?a=blobdiff_plain;f=conductor%2Fconductor%2Fcontroller%2Ftranslator.py;h=8184639f4317c91a8729fa0410ce2b47bbeb2637;hb=41de0bc00450d43af6ad68d7d74eec66ef4ffd7e;hp=dbff2d2fd238eec71ba3cfcf70752295dab792c0;hpb=1c4a9f7f0d7ca61df2f0955f60aea9670e399d45;p=optf%2Fhas.git diff --git a/conductor/conductor/controller/translator.py b/conductor/conductor/controller/translator.py index dbff2d2..8184639 100644 --- a/conductor/conductor/controller/translator.py +++ b/conductor/conductor/controller/translator.py @@ -22,34 +22,37 @@ import datetime import json import os import uuid -import yaml -from oslo_config import cfg -from oslo_log import log import six - +import yaml from conductor import __file__ as conductor_root -from conductor.common.music import messaging as music_messaging -from conductor.common import threshold from conductor import messaging from conductor import service +from conductor.common import threshold +from conductor.common.music import messaging as music_messaging +from conductor.data.plugins.triage_translator.triage_translator_data import TraigeTranslatorData +from conductor.data.plugins.triage_translator.triage_translator import TraigeTranslator +from oslo_config import cfg +from oslo_log import log + LOG = log.getLogger(__name__) CONF = cfg.CONF -VERSIONS = ["2016-11-01", "2017-10-10"] +VERSIONS = ["2016-11-01", "2017-10-10", "2018-02-01"] LOCATION_KEYS = ['latitude', 'longitude', 'host_name', 'clli_code'] INVENTORY_PROVIDERS = ['aai'] -INVENTORY_TYPES = ['cloud', 'service', 'transport'] +INVENTORY_TYPES = ['cloud', 'service', 'transport', 'vfmodule'] DEFAULT_INVENTORY_PROVIDER = INVENTORY_PROVIDERS[0] -CANDIDATE_KEYS = ['inventory_type', 'candidate_id', 'location_id', - 'location_type', 'cost'] -DEMAND_KEYS = ['inventory_provider', 'inventory_type', 'service_type', - 'service_id', 'service_resource_id', 'customer_id', - 'default_cost', 'candidates', 'region', 'complex', - 'required_candidates', 'excluded_candidates', - 'existing_placement', 'subdivision', 'flavor', 'attributes'] +CANDIDATE_KEYS = ['candidate_id', 'cost', 'inventory_type', 'location_id', + 'location_type'] +DEMAND_KEYS = ['filtering_attributes', 'passthrough_attributes', 'candidates', 'complex', 'conflict_identifier', + 'customer_id', 'default_cost', 'excluded_candidates', + 'existing_placement', 'flavor', 'inventory_provider', + 'inventory_type', 'port_key', 'region', 'required_candidates', + 'service_id', 'service_resource_id', 'service_subscription', + 'service_type', 'subdivision', 'unique', 'vlan_key'] CONSTRAINT_KEYS = ['type', 'demands', 'properties'] CONSTRAINTS = { # constraint_type: { @@ -95,7 +98,21 @@ CONSTRAINTS = { 'category': ['disaster', 'region', 'complex', 'country', 'time', 'maintenance']}, }, + 'vim_fit': { + 'split': True, + 'required': ['controller'], + 'optional': ['request'], + }, + 'hpa': { + 'split': True, + 'required': ['evaluate'], + }, } +HPA_FEATURES = ['architecture', 'hpa-feature', 'hpa-feature-attributes', + 'hpa-version', 'mandatory', 'directives'] +HPA_OPTIONAL = ['score'] +HPA_ATTRIBUTES = ['hpa-attribute-key', 'hpa-attribute-value', 'operator'] +HPA_ATTRIBUTES_OPTIONAL = ['unit'] class TranslatorException(Exception): @@ -120,7 +137,8 @@ class Translator(object): self._translation = None self._valid = False self._ok = False - + self.triageTranslatorData= TraigeTranslatorData() + self.triageTranslator = TraigeTranslator() # Set up the RPC service(s) we want to talk to. self.data_service = self.setup_rpc(self.conf, "data") @@ -207,8 +225,8 @@ class Translator(object): "{} {} has an invalid key {}".format( name, content_name, key)) - demand_keys = self._demands.keys() - location_keys = self._locations.keys() + demand_keys = list(self._demands.keys()) # Python 3 Conversion -- dict object to list object + location_keys = list(self._locations.keys()) # Python 3 Conversion -- dict object to list object for constraint_name, constraint in self._constraints.items(): # Require a single demand (string), or a list of one or more. @@ -261,7 +279,7 @@ class Translator(object): # Traverse a dict elif type(obj) is dict: # Did we find a "{get_param: ...}" intrinsic? - if obj.keys() == ['get_param']: + if list(obj.keys()) == ['get_param']: param_name = obj['get_param'] # The parameter name must be a string. @@ -282,7 +300,7 @@ class Translator(object): return self._parameters.get(param_name) # Not an intrinsic. Traverse as usual. - for key in obj.keys(): + for key in list(obj.keys()): # Add path to the breadcrumb trail. new_path = list(path) new_path.append(key) @@ -375,7 +393,7 @@ class Translator(object): "not a dictionary".format(name)) # Must have only supported keys - for key in candidate.keys(): + for key in list(candidate.keys()): if key not in CANDIDATE_KEYS: raise TranslatorException( "Candidate with invalid key {} found " @@ -427,14 +445,14 @@ class Translator(object): "demand {}".format(inventory_type, name) ) - # For service inventories, customer_id and + # For service and vfmodule inventories, customer_id and # service_type MUST be specified - if inventory_type == 'service': - attributes = requirement.get('attributes') + if inventory_type == 'service' or inventory_type == 'vfmodule': + filtering_attributes = requirement.get('filtering_attributes') - if attributes: - customer_id = attributes.get('customer-id') - global_customer_id = attributes.get('global-customer-id') + if filtering_attributes: + customer_id = filtering_attributes.get('customer-id') + global_customer_id = filtering_attributes.get('global-customer-id') if global_customer_id: customer_id = global_customer_id else: @@ -447,7 +465,7 @@ class Translator(object): "Customer ID not specified for " "demand {}".format(name) ) - if not attributes and not service_type: + if not filtering_attributes and not service_type: raise TranslatorException( "Service Type not specified for " "demand {}".format(name) @@ -480,7 +498,13 @@ class Translator(object): args = { "demands": { name: requirements, - } + }, + "plan_info":{ + "plan_id": self._plan_id, + "plan_name": self._plan_name + }, + "triage_translator_data": self.triageTranslatorData.__dict__ + } # Check if required_candidate and excluded candidate @@ -499,7 +523,6 @@ class Translator(object): " list are not mutually exclusive for demand" " {}".format(name) ) - response = self.data_service.call( ctxt=ctxt, method="resolve_demands", @@ -507,10 +530,13 @@ class Translator(object): resolved_demands = \ response and response.get('resolved_demands') + triage_data_trans = \ + response and response.get('trans') - required_candidates = resolved_demands\ + required_candidates = resolved_demands \ .get('required_candidates') if not resolved_demands: + self.triageTranslator.thefinalCallTrans(triage_data_trans) raise TranslatorException( "Unable to resolve inventory " "candidates for demand {}" @@ -521,11 +547,13 @@ class Translator(object): inventory_candidates.append(candidate) if len(inventory_candidates) < 1: if not required_candidates: + self.triageTranslator.thefinalCallTrans(triage_data_trans) raise TranslatorException( "Unable to find any candidate for " "demand {}".format(name) ) else: + self.triageTranslator.thefinalCallTrans(triage_data_trans) raise TranslatorException( "Unable to find any required " "candidate for demand {}" @@ -534,9 +562,64 @@ class Translator(object): parsed[name] = { "candidates": inventory_candidates, } - + self.triageTranslator.thefinalCallTrans(triage_data_trans) return parsed + def validate_hpa_constraints(self, req_prop, value): + for para in value.get(req_prop): + # Make sure there is at least one + # set of id, type, directives and flavorProperties + if not para.get('id') \ + or not para.get('type') \ + or not para.get('directives') \ + or not para.get('flavorProperties') \ + or para.get('id') == '' \ + or para.get('type') == '' \ + or not isinstance(para.get('directives'), list) \ + or para.get('flavorProperties') == '': + raise TranslatorException( + "HPA requirements need at least " + "one set of id, type, directives and flavorProperties" + ) + for feature in para.get('flavorProperties'): + if type(feature) is not dict: + raise TranslatorException("HPA feature must be a dict") + # process mandatory parameter + hpa_mandatory = set(HPA_FEATURES).difference(feature.keys()) + if bool(hpa_mandatory): + raise TranslatorException( + "Lack of compulsory elements inside HPA feature") + # process optional parameter + hpa_optional = set(feature.keys()).difference(HPA_FEATURES) + if hpa_optional and not hpa_optional.issubset(HPA_OPTIONAL): + raise TranslatorException( + "Got unrecognized elements inside HPA feature") + if feature.get('mandatory') == 'False' and not feature.get( + 'score'): + raise TranslatorException( + "Score needs to be present if mandatory is False") + + for attr in feature.get('hpa-feature-attributes'): + if type(attr) is not dict: + raise TranslatorException( + "HPA feature attributes must be a dict") + + # process mandatory hpa attribute parameter + hpa_attr_mandatory = set(HPA_ATTRIBUTES).difference( + attr.keys()) + if bool(hpa_attr_mandatory): + raise TranslatorException( + "Lack of compulsory elements inside HPA " + "feature attributes") + # process optional hpa attribute parameter + hpa_attr_optional = set(attr.keys()).difference( + HPA_ATTRIBUTES) + if hpa_attr_optional and not hpa_attr_optional.issubset( + HPA_ATTRIBUTES_OPTIONAL): + raise TranslatorException( + "Invalid attributes '{}' found inside HPA " + "feature attributes".format(hpa_attr_optional)) + def parse_constraints(self, constraints): """Validate/prepare constraints for use by the solver.""" if not isinstance(constraints, dict): @@ -574,7 +657,7 @@ class Translator(object): # Make sure all required properties are present required = constraint_def.get('required', []) for req_prop in required: - if req_prop not in value.keys(): + if req_prop not in list(value.keys()): raise TranslatorException( "Required property '{}' not found in " "constraint named '{}'".format( @@ -585,10 +668,13 @@ class Translator(object): "No value specified for property '{}' in " "constraint named '{}'".format( req_prop, name)) + # For HPA constraints + if constraint_type == 'hpa': + self.validate_hpa_constraints(req_prop, value) # Make sure there are no unknown properties optional = constraint_def.get('optional', []) - for prop_name in value.keys(): + for prop_name in list(value.keys()): if prop_name not in required + optional: raise TranslatorException( "Unknown property '{}' in " @@ -599,7 +685,7 @@ class Translator(object): # sure its value is one of the allowed ones. allowed = constraint_def.get('allowed', {}) for prop_name, allowed_values in allowed.items(): - if prop_name in value.keys(): + if prop_name in list(value.keys()): prop_value = value.get(prop_name, '') if prop_value not in allowed_values: raise TranslatorException( @@ -611,7 +697,7 @@ class Translator(object): # Break all threshold-formatted values into parts thresholds = constraint_def.get('thresholds', {}) for thr_prop, base_units in thresholds.items(): - if thr_prop in value.keys(): + if thr_prop in list(value.keys()): expression = value.get(thr_prop) thr = threshold.Threshold(expression, base_units) value[thr_prop] = thr.parts @@ -665,12 +751,12 @@ class Translator(object): if type(optimization_copy) is not dict: raise TranslatorException("Optimization must be a dictionary.") - goals = optimization_copy.keys() + goals = list(optimization_copy.keys()) if goals != ['minimize']: raise TranslatorException( "Optimization must contain a single goal of 'minimize'.") - funcs = optimization_copy['minimize'].keys() + funcs = list(optimization_copy['minimize'].keys()) if funcs != ['sum']: raise TranslatorException( "Optimization goal 'minimize' must " @@ -683,6 +769,28 @@ class Translator(object): "Optimization goal 'minimize', function 'sum' " "must be a list of exactly two operands.") + def get_latency_between_args(operand): + args = operand.get('latency_between') + if type(args) is not list and len(args) != 2: + raise TranslatorException( + "Optimization 'latency_between' arguments must " + "be a list of length two.") + + got_demand = False + got_location = False + for arg in args: + if not got_demand and arg in list(self._demands.keys()): + got_demand = True + if not got_location and arg in list(self._locations.keys()): + got_location = True + if not got_demand or not got_location: + raise TranslatorException( + "Optimization 'latency_between' arguments {} must " + "include one valid demand name and one valid " + "location name.".format(args)) + + return args + def get_distance_between_args(operand): args = operand.get('distance_between') if type(args) is not list and len(args) != 2: @@ -693,9 +801,9 @@ class Translator(object): got_demand = False got_location = False for arg in args: - if not got_demand and arg in self._demands.keys(): + if not got_demand and arg in list(self._demands.keys()): got_demand = True - if not got_location and arg in self._locations.keys(): + if not got_location and arg in list(self._locations.keys()): got_location = True if not got_demand or not got_location: raise TranslatorException( @@ -710,34 +818,47 @@ class Translator(object): args = None nested = False - if operand.keys() == ['distance_between']: + if list(operand.keys()) == ['distance_between']: # Value must be a list of length 2 with one # location and one demand function = 'distance_between' args = get_distance_between_args(operand) - elif operand.keys() == ['product']: + elif list(operand.keys()) == ['product']: for product_op in operand['product']: if threshold.is_number(product_op): weight = product_op - elif type(product_op) is dict: - if product_op.keys() == ['distance_between']: + elif isinstance(product_op, dict): + if list(product_op.keys()) == ['latency_between']: + function = 'latency_between' + args = get_latency_between_args(product_op) + elif list(product_op.keys()) == ['distance_between']: function = 'distance_between' args = get_distance_between_args(product_op) - elif product_op.keys() == ['aic_version']: + elif list(product_op.keys()) == ['aic_version']: function = 'aic_version' args = product_op.get('aic_version') - elif product_op.keys() == ['sum']: + elif list(product_op.keys()) == ['hpa_score']: + function = 'hpa_score' + args = product_op.get('hpa_score') + if not self.is_hpa_policy_exists(args): + raise TranslatorException( + "HPA Score Optimization must include a " + "HPA Policy constraint ") + elif list(product_op.keys()) == ['sum']: nested = True nested_operands = product_op.get('sum') for nested_operand in nested_operands: - if nested_operand.keys() == ['product']: + if list(nested_operand.keys()) == ['product']: nested_weight = weight for nested_product_op in nested_operand['product']: if threshold.is_number(nested_product_op): nested_weight = nested_weight * int(nested_product_op) - elif type(nested_product_op) is dict: - if nested_product_op.keys() == ['distance_between']: + elif isinstance(nested_product_op, dict): + if list(nested_product_op.keys()) == ['latency_between']: + function = 'latency_between' + args = get_latency_between_args(nested_product_op) + elif list(nested_product_op.keys()) == ['distance_between']: function = 'distance_between' args = get_distance_between_args(nested_product_op) parsed['operands'].append( @@ -749,16 +870,6 @@ class Translator(object): } ) - elif type(product_op) is unicode: - if product_op == 'W1': - # get this weight from configuration file - weight = self.conf.controller.weight1 - elif product_op == 'W2': - # get this weight from configuration file - weight = self.conf.controller.weight2 - elif product_op == 'cost': - function = 'cost' - if not args: raise TranslatorException( "Optimization products must include at least " @@ -777,25 +888,45 @@ class Translator(object): ) return parsed + def is_hpa_policy_exists(self, demand_list): + # Check if a HPA constraint exist for the demands in the demand list. + constraints_copy = copy.deepcopy(self._constraints) + for demand in demand_list: + for name, constraint in constraints_copy.items(): + constraint_type = constraint.get('type') + if constraint_type == 'hpa': + hpa_demands = constraint.get('demands') + if demand in hpa_demands: + return True + return False + def parse_reservations(self, reservations): demands = self._demands - if type(reservations) is not dict: + if not isinstance(reservations, dict): raise TranslatorException("Reservations must be provided in " "dictionary form") - parsed = {} if reservations: parsed['counter'] = 0 - for name, reservation in reservations.items(): - if not reservation.get('properties'): - reservation['properties'] = {} - for demand in reservation.get('demands', []): - if demand in demands.keys(): - constraint_demand = name + '_' + demand - parsed['demands'] = {} - parsed['demands'][constraint_demand] = copy.deepcopy(reservation) - parsed['demands'][constraint_demand]['name'] = name - parsed['demands'][constraint_demand]['demand'] = demand + parsed['demands'] = {} + + for key, value in reservations.items(): + + if key == "service_model": + parsed['service_model'] = value + + elif key == "service_candidates": + for name, reservation_details in value.items(): + if not reservation_details.get('properties'): + reservation_details['properties'] = {} + for demand in reservation_details.get('demands', []): + if demand in list(demands.keys()): + reservation_demand = name + '_' + demand + parsed['demands'][reservation_demand] = copy.deepcopy(reservation_details) + parsed['demands'][reservation_demand]['name'] = name + parsed['demands'][reservation_demand]['demands'] = demand + else: + raise TranslatorException("Demand {} must be provided in demands section".format(demand)) return parsed @@ -804,7 +935,9 @@ class Translator(object): if not self.valid: raise TranslatorException("Can't translate an invalid template.") - request_type = self._parameters.get("request_type") or "" + request_type = self._parameters.get("request_type") \ + or self._parameters.get("REQUEST_TYPE") \ + or "" self._translation = { "conductor_solver": { @@ -830,7 +963,7 @@ class Translator(object): self.do_translation() self._ok = True except Exception as exc: - self._error_message = exc.message + self._error_message = exc.args @property def valid(self):