#!/usr/bin/env python # # ------------------------------------------------------------------------- # Copyright (c) 2015-2017 AT&T Intellectual Property # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # ------------------------------------------------------------------------- # # import json import collections import operator import random from conductor.solver.optimizer.constraints \ import access_distance as access_dist from conductor.solver.optimizer.constraints \ import aic_distance as aic_dist from conductor.solver.optimizer.constraints \ import attribute as attribute_constraint from conductor.solver.optimizer.constraints import hpa from conductor.solver.optimizer.constraints \ import inventory_group from conductor.solver.optimizer.constraints \ import service as service_constraint from conductor.solver.optimizer.constraints import vim_fit from conductor.solver.optimizer.constraints import zone from conductor.solver.request import demand from conductor.solver.request import objective from conductor.solver.request.functions import aic_version from conductor.solver.request.functions import cost from conductor.solver.request.functions import distance_between from conductor.solver.request.functions import hpa_score from conductor.solver.request.functions import latency_between from conductor.solver.request import objective from conductor.solver.triage_tool.traige_latency import TriageLatency from oslo_log import log LOG = log.getLogger(__name__) # FIXME(snarayanan): This is really a SolverRequest (or Request) object class Parser(object): demands = None # type: Dict[Any, Any] locations = None # type: Dict[Any, Any] obj_func_param = None def __init__(self, _region_gen=None): self.demands = {} self.locations = {} self.region_gen = _region_gen self.constraints = {} self.objective = None self.obj_func_param = list() self.cei = None self.request_id = None self.request_type = None self.region_group = None # def get_data_engine_interface(self): # self.cei = cei.ConstraintEngineInterface() # FIXME(snarayanan): This should just be parse_template def parse_template(self, json_template=None, country_groups=None, regions_maps=None): if json_template is None: LOG.error("No template specified") return "Error" # get request type self.request_type = json_template['conductor_solver']['request_type'] # get demands demand_list = json_template["conductor_solver"]["demands"] for demand_id, candidate_list in demand_list.items(): current_demand = demand.Demand(demand_id) # candidate should only have minimal information like location_id for candidate in candidate_list["candidates"]: candidate_id = candidate["candidate_id"] current_demand.resources[candidate_id] = candidate current_demand.sort_base = 0 # this is only for testing self.demands[demand_id] = current_demand # get locations location_list = json_template["conductor_solver"]["locations"] for location_id, location_info in location_list.items(): loc = demand.Location(location_id) loc.loc_type = "coordinates" loc.value = (float(location_info["latitude"]), float(location_info["longitude"])) loc.country = location_info[ 'country'] if 'country' in location_info else None self.locations[location_id] = loc # get constraints input_constraints = json_template["conductor_solver"]["constraints"] for constraint_id, constraint_info in input_constraints.items(): constraint_type = constraint_info["type"] constraint_demands = list() parsed_demands = constraint_info["demands"] if isinstance(parsed_demands, list): for d in parsed_demands: constraint_demands.append(d) else: constraint_demands.append(parsed_demands) if constraint_type == "distance_to_location": c_property = constraint_info.get("properties") location_id = c_property.get("location") op = operator.le # default operator c_op = c_property.get("distance").get("operator") if c_op == ">": op = operator.gt elif c_op == ">=": op = operator.ge elif c_op == "<": op = operator.lt elif c_op == "<=": op = operator.le elif c_op == "=": op = operator.eq dist_value = c_property.get("distance").get("value") my_access_distance_constraint = access_dist.AccessDistance( constraint_id, constraint_type, constraint_demands, _comparison_operator=op, _threshold=dist_value, _location=self.locations[location_id]) self.constraints[my_access_distance_constraint.name] = \ my_access_distance_constraint elif constraint_type == "distance_between_demands": c_property = constraint_info.get("properties") op = operator.le # default operator c_op = c_property.get("distance").get("operator") if c_op == ">": op = operator.gt elif c_op == ">=": op = operator.ge elif c_op == "<": op = operator.lt elif c_op == "<=": op = operator.le elif c_op == "=": op = operator.eq dist_value = c_property.get("distance").get("value") my_aic_distance_constraint = aic_dist.AICDistance( constraint_id, constraint_type, constraint_demands, _comparison_operator=op, _threshold=dist_value) self.constraints[my_aic_distance_constraint.name] = \ my_aic_distance_constraint elif constraint_type == "inventory_group": my_inventory_group_constraint = \ inventory_group.InventoryGroup( constraint_id, constraint_type, constraint_demands) self.constraints[my_inventory_group_constraint.name] = \ my_inventory_group_constraint elif constraint_type == "region_fit": c_property = constraint_info.get("properties") controller = c_property.get("controller") request = c_property.get("request") # inventory type is cloud for region_fit inventory_type = "cloud" my_service_constraint = service_constraint.Service( constraint_id, constraint_type, constraint_demands, _controller=controller, _request=request, _cost=None, _inventory_type=inventory_type) self.constraints[my_service_constraint.name] = \ my_service_constraint elif constraint_type == "instance_fit": c_property = constraint_info.get("properties") controller = c_property.get("controller") request = c_property.get("request") # inventory type is service for instance_fit inventory_type = "service" my_service_constraint = service_constraint.Service( constraint_id, constraint_type, constraint_demands, _controller=controller, _request=request, _cost=None, _inventory_type=inventory_type) self.constraints[my_service_constraint.name] = \ my_service_constraint elif constraint_type == "zone": c_property = constraint_info.get("properties") location_id = c_property.get("location") qualifier = c_property.get("qualifier") category = c_property.get("category") location = self.locations[location_id] if location_id else None my_zone_constraint = zone.Zone( constraint_id, constraint_type, constraint_demands, _qualifier=qualifier, _category=category, _location=location) self.constraints[my_zone_constraint.name] = my_zone_constraint elif constraint_type == "attribute": c_property = constraint_info.get("properties") my_attribute_constraint = \ attribute_constraint.Attribute(constraint_id, constraint_type, constraint_demands, _properties=c_property) self.constraints[my_attribute_constraint.name] = \ my_attribute_constraint elif constraint_type == "hpa": LOG.debug("Creating constraint - {}".format(constraint_type)) c_property = constraint_info.get("properties") my_hpa_constraint = hpa.HPA(constraint_id, constraint_type, constraint_demands, _properties=c_property) self.constraints[my_hpa_constraint.name] = my_hpa_constraint elif constraint_type == "vim_fit": LOG.debug("Creating constraint - {}".format(constraint_type)) c_property = constraint_info.get("properties") my_vim_constraint = vim_fit.VimFit(constraint_id, constraint_type, constraint_demands, _properties=c_property) self.constraints[my_vim_constraint.name] = my_vim_constraint else: LOG.error("unknown constraint type {}".format(constraint_type)) return # get objective function if "objective" not in json_template["conductor_solver"] \ or not json_template["conductor_solver"]["objective"]: self.objective = objective.Objective() else: input_objective = json_template["conductor_solver"]["objective"] self.objective = objective.Objective() self.objective.goal = input_objective["goal"] self.objective.operation = input_objective["operation"] self.latencyTriage = TriageLatency() LOG.info("objective function params") for operand_data in input_objective["operands"]: if operand_data["function"] == "latency_between": self.obj_func_param.append(operand_data["function_param"][1]) LOG.info("done - objective function params") for operand_data in input_objective["operands"]: operand = objective.Operand() operand.operation = operand_data["operation"] operand.weight = float(operand_data["weight"]) if operand_data["function"] == "latency_between": LOG.debug("Processing objective function latency_between") self.latencyTriage.takeOpimaztionType(operand_data["function"]) func = latency_between.LatencyBetween("latency_between") func.region_group = self.assign_region_group_weight(country_groups, regions_maps) param = operand_data["function_param"][0] if param in self.locations: func.loc_a = self.locations[param] elif param in self.demands: func.loc_a = self.demands[param] param = operand_data["function_param"][1] if param in self.locations: func.loc_z = self.locations[param] elif param in self.demands: func.loc_z = self.demands[param] operand.function = func elif operand_data["function"] == "distance_between": LOG.debug("Processing objective function distance_between") self.latencyTriage.takeOpimaztionType(operand_data["function"]) func = distance_between.DistanceBetween("distance_between") param = operand_data["function_param"][0] if param in self.locations: func.loc_a = self.locations[param] elif param in self.demands: func.loc_a = self.demands[param] param = operand_data["function_param"][1] if param in self.locations: func.loc_z = self.locations[param] elif param in self.demands: func.loc_z = self.demands[param] operand.function = func elif operand_data["function"] == "aic_version": self.objective.goal = "min_aic" func = aic_version.AICVersion("aic_version") func.loc = operand_data["function_param"] operand.function = func elif operand_data["function"] == "cost": func = cost.Cost("cost") func.loc = operand_data["function_param"] operand.function = func elif operand_data["function"] == "hpa_score": func = hpa_score.HPAScore("hpa_score") operand.function = func self.objective.operand_list.append(operand) self.latencyTriage.updateTriageLatencyDB(self.plan_id, self.request_id) def assign_region_group_weight(self, countries, regions): """ assign the latency group value to the country and returns a map""" LOG.info("Processing Assigning Latency Weight to Countries ") countries = self.resolve_countries(countries, regions, self.get_candidate_country_list()) # resolve the countries based on region type region_latency_weight = collections.OrderedDict() weight = 0 if countries is None: LOG.info("No countries available to assign latency weight ") return region_latency_weight try: l_weight = '' for i, e in enumerate(countries): if e is None: continue for k, x in enumerate(e.split(',')): region_latency_weight[x] = weight l_weight += x + " : " + str(weight) l_weight += ',' weight = weight + 1 LOG.info("Latency Weights Assigned ") LOG.info(l_weight) except Exception as err: LOG.info("Exception while assigning the latency weights " + err) print(err) return region_latency_weight def get_candidate_country_list(self): LOG.info("Processing Get Candidate Countries from demands ") candidate_country_list = list() try: candidate_countries = '' for demand_id, demands in self.demands.items(): candidate_countries += demand_id for candidte in list(demands.resources.values()): # Python 3 Conversion -- dict object to list object candidate_country_list.append(candidte["country"]) candidate_countries += candidte["country"] candidate_countries += ',' LOG.info("Available Candidate Countries " + candidate_countries) except Exception as err: print(err) return candidate_country_list def resolve_countries(self, countries_list, regions_map, candidates_country_list): # check the map with given location and retrieve the values LOG.info("Resolving Countries ") if countries_list is None: LOG.info("No Countries are available ") return countries_list countries_list = self.filter_invalid_rules(countries_list, regions_map) if countries_list is not None and countries_list.__len__() > 0 and countries_list.__getitem__( countries_list.__len__() - 1) == "*": self.process_wildcard_rules(candidates_country_list, countries_list) else: self.process_without_wildcard_rules(candidates_country_list, countries_list) return countries_list def process_without_wildcard_rules(self, candidates_country_list, countries_list): try: temp_country_list = list() for country_group in countries_list: for country in country_group.split(','): temp_country_list.append(country) tmpcl = '' for cl in temp_country_list: tmpcl += cl tmpcl += ',' LOG.info("Countries List before diff " + tmpcl) ccl = '' for cl in candidates_country_list: ccl += cl ccl += ',' LOG.info("Candidates Countries List before diff " + ccl) # filterout the candidates that does not match the countries list # filter(lambda x: x not in countries_list, self.get_candidate_countries_list()) LOG.info("Processing Difference between Candidate Countries and Latency Countries without *. ") diff_bw_candidates_and_countries = list(set(candidates_country_list).difference(temp_country_list)) candcl = '' for cl in diff_bw_candidates_and_countries: candcl += cl candcl += ',' LOG.info("Available countries after processing diff between " + candcl) self.drop_no_latency_rule_candidates(diff_bw_candidates_and_countries) except Exception as error: print(error) def drop_no_latency_rule_candidates(self, diff_bw_candidates_and_countries): cadidate_list_ = list() temp_candidates = dict() for demand_id, demands in self.demands.items(): LOG.info("demand id " + demand_id) for candidte in list(demands.resources.values()): # Python 3 Conversion -- dict object to list object LOG.info("candidate id " + candidte['candidate_id']) dem_candidate = {demand_id: demands} temp_candidates.update(dem_candidate) droped_candidates = '' for demand_id, demands in temp_candidates.items(): droped_candidates += demand_id for candidate in list(demands.resources.values()): # Python 3 Conversion -- dict object to list object if demand_id in self.obj_func_param and candidate["country"] in diff_bw_candidates_and_countries: droped_candidates += candidate['candidate_id'] droped_candidates += ',' self.latencyTriage.latencyDroppedCandiate(candidate['candidate_id'], demand_id, reason="diff_bw_candidates_and_countries,Latecy weight ") self.demands[demand_id].resources.pop(candidate['candidate_id']) LOG.info("dropped " + droped_candidates) # for demand_id, candidate_list in self.demands: # LOG.info("Candidates for demand " + demand_id) # cadidate_list_ = self.demands[demand_id]['candidates'] # droped_candidates = '' # xlen = cadidate_list_.__len__() - 1 # len = xlen # # LOG.info("Candidate List Length "+str(len)) # for i in range(len + 1): # # LOG.info("iteration " + i) # LOG.info("Candidate Country " + cadidate_list_[xlen]["country"]) # if cadidate_list_[xlen]["country"] in diff_bw_candidates_and_countries: # droped_candidates += cadidate_list_[xlen]["country"] # droped_candidates += ',' # self.demands[demand_id]['candidates'].remove(cadidate_list_[xlen]) # # filter(lambda candidate: candidate in candidate_list["candidates"]) # # LOG.info("Droping Cadidate not eligible for latency weight. Candidate ID " + cadidate_list_[xlen]["candidate_id"] + " Candidate Country: "+cadidate_list_[xlen]["country"]) # xlen = xlen - 1 # if xlen < 0: break # LOG.info("Dropped Candidate Countries " + droped_candidates + " from demand " + demand_id) def process_wildcard_rules(self, candidates_country_list, countries_list, ): LOG.info("Processing the rules for " + countries_list.__getitem__(countries_list.__len__() - 1)) candidate_countries = '' countries_list.remove(countries_list.__getitem__( countries_list.__len__() - 1)) # remove the wildcard and replace it with available candidates countries temp_country_list = list() for country_group in countries_list: for country in country_group.split(','): temp_country_list.append(country) temp_countries = '' for cl in temp_country_list: temp_countries += cl temp_countries += ',' LOG.info("Countries before diff " + temp_countries) ccl = '' for cl in candidates_country_list: ccl += cl ccl += ',' LOG.info("Candidates Countries List before diff " + ccl) diff_bw_candidates_and_countries = list(set(candidates_country_list).difference(temp_country_list)) LOG.info("Processing Difference between Candidate Countries and Latency Countries for * . ") for c_group in diff_bw_candidates_and_countries: candidate_countries += c_group candidate_countries += ',' LOG.info("Difference: " + candidate_countries[:-1]) if candidate_countries.__len__() > 0: LOG.info(candidate_countries[:-1]) countries_list.append(candidate_countries[:-1]) # append the list of difference to existing countries ac = '' for cl in countries_list: ac += cl ac += ',' LOG.info("Available countries after processing diff between " + ac) def filter_invalid_rules(self, countries_list, regions_map): invalid_rules = list(); for i, e in enumerate(countries_list): if e is None: continue for k, region in enumerate(e.split(',')): LOG.info("Processing the Rule for " + region) if region.__len__() != 3: if region == "*": continue region_list = regions_map.get(region) if region_list is None: LOG.info("Invalid region " + region) invalid_rules.append(region) continue countries_list.remove(countries_list[i]) countries_list.insert(i, region_list) for ir in invalid_rules: LOG.info("Filtering out invalid rules from countries list ") LOG.info("invalid rule " + ir) countries_list = list(filter(lambda country: (country not in invalid_rules), countries_list)) available_countries = '' for cl in countries_list: available_countries += cl available_countries += ',' LOG.info("Available countries after the filteration " + available_countries[:-1]) return countries_list def reorder_constraint(self): # added manual ranking to the constraint type for optimizing purpose the last 2 are costly interaction for constraint_name, constraint in self.constraints.items(): if constraint.constraint_type == "distance_to_location": constraint.rank = 1 elif constraint.constraint_type == "zone": constraint.rank = 2 elif constraint.constraint_type == "attribute": constraint.rank = 3 elif constraint.constraint_type == "hpa": constraint.rank = 4 elif constraint.constraint_type == "inventory_group": constraint.rank = 5 elif constraint.constraint_type == "vim_fit": constraint.rank = 6 elif constraint.constraint_type == "instance_fit": constraint.rank = 7 elif constraint.constraint_type == "region_fit": constraint.rank = 8 else: constraint.rank = 9 def attr_sort(self, attrs=['rank']): # this helper for sorting the rank return lambda k: [getattr(k, attr) for attr in attrs] def sort_constraint_by_rank(self): # this sorts as rank for d_name, cl in self.demands.items(): cl_list = cl.constraint_list cl_list.sort(key=self.attr_sort(attrs=['rank'])) def assgin_constraints_to_demands(self): # spread the constraints over the demands self.reorder_constraint() for constraint_name, constraint in self.constraints.items(): for d in constraint.demand_list: if d in list(self.demands.keys()): # Python 3 Conversion -- dict object to list object self.demands[d].constraint_list.append(constraint) self.sort_constraint_by_rank()