conductor/conductor/solver/request/parser.py

   1 #!/usr/bin/env python
   2 #
   3 # -------------------------------------------------------------------------
   4 #   Copyright (c) 2015-2017 AT&T Intellectual Property
   5 #   Copyright (C) 2020 Wipro Limited.
   6 #
   7 #   Licensed under the Apache License, Version 2.0 (the "License");
   8 #   you may not use this file except in compliance with the License.
   9 #   You may obtain a copy of the License at
  10 #
  11 #       http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 #   Unless required by applicable law or agreed to in writing, software
  14 #   distributed under the License is distributed on an "AS IS" BASIS,
  15 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 #   See the License for the specific language governing permissions and
  17 #   limitations under the License.
  18 #
  19 # -------------------------------------------------------------------------
  20 #
  21
  22
  23 # import json
  24 import collections
  25 import operator
  26 import random
  27
  28 from conductor.solver.optimizer.constraints \
  29     import access_distance as access_dist
  30 from conductor.solver.optimizer.constraints \
  31     import aic_distance as aic_dist
  32 from conductor.solver.optimizer.constraints \
  33     import attribute as attribute_constraint
  34 from conductor.solver.optimizer.constraints import hpa
  35 from conductor.solver.optimizer.constraints \
  36     import inventory_group
  37 from conductor.solver.optimizer.constraints \
  38     import service as service_constraint
  39 from conductor.solver.optimizer.constraints import vim_fit
  40 from conductor.solver.optimizer.constraints import zone
  41 from conductor.solver.optimizer.constraints import threshold
  42 from conductor.solver.request import demand
  43 from conductor.solver.request import objective
  44 from conductor.solver.request.functions import aic_version
  45 from conductor.solver.request.functions import cost
  46 from conductor.solver.request.functions import distance_between
  47 from conductor.solver.request.functions import hpa_score
  48 from conductor.solver.request.functions import latency_between
  49 from conductor.solver.request import objective
  50 from conductor.solver.triage_tool.traige_latency import TriageLatency
  51 from oslo_log import log
  52
  53 LOG = log.getLogger(__name__)
  54
  55
  56 # FIXME(snarayanan): This is really a SolverRequest (or Request) object
  57 class Parser(object):
  58
  59     demands = None  # type: Dict[Any, Any]
  60     locations = None  # type: Dict[Any, Any]
  61     obj_func_param = None
  62
  63     def __init__(self, _region_gen=None):
  64         self.demands = {}
  65         self.locations = {}
  66         self.region_gen = _region_gen
  67         self.constraints = {}
  68         self.objective = None
  69         self.obj_func_param = list()
  70         self.cei = None
  71         self.request_id = None
  72         self.request_type = None
  73         self.region_group = None
  74
  75     # def get_data_engine_interface(self):
  76     #    self.cei = cei.ConstraintEngineInterface()
  77
  78     # FIXME(snarayanan): This should just be parse_template
  79     def parse_template(self, json_template=None, country_groups=None, regions_maps=None):
  80         if json_template is None:
  81             LOG.error("No template specified")
  82             return "Error"
  83
  84         # get request type
  85         self.request_type = json_template['conductor_solver']['request_type']
  86
  87         # get demands
  88         demand_list = json_template["conductor_solver"]["demands"]
  89         for demand_id, candidate_list in demand_list.items():
  90             current_demand = demand.Demand(demand_id)
  91             # candidate should only have minimal information like location_id
  92             for candidate in candidate_list["candidates"]:
  93                 candidate_id = candidate["candidate_id"]
  94                 current_demand.resources[candidate_id] = candidate
  95             current_demand.sort_base = 0  # this is only for testing
  96             self.demands[demand_id] = current_demand
  97
  98         # get locations
  99         location_list = json_template["conductor_solver"]["locations"]
 100         for location_id, location_info in location_list.items():
 101             loc = demand.Location(location_id)
 102             loc.loc_type = "coordinates"
 103             loc.value = (float(location_info["latitude"]),
 104                          float(location_info["longitude"]))
 105             loc.country = location_info[
 106                 'country'] if 'country' in location_info else None
 107             self.locations[location_id] = loc
 108
 109         # get constraints
 110         input_constraints = json_template["conductor_solver"]["constraints"]
 111         for constraint_id, constraint_info in input_constraints.items():
 112             constraint_type = constraint_info["type"]
 113             constraint_demands = list()
 114             parsed_demands = constraint_info["demands"]
 115             if isinstance(parsed_demands, list):
 116                 for d in parsed_demands:
 117                     constraint_demands.append(d)
 118             else:
 119                 constraint_demands.append(parsed_demands)
 120             if constraint_type == "distance_to_location":
 121                 c_property = constraint_info.get("properties")
 122                 location_id = c_property.get("location")
 123                 op = operator.le  # default operator
 124                 c_op = c_property.get("distance").get("operator")
 125                 if c_op == ">":
 126                     op = operator.gt
 127                 elif c_op == ">=":
 128                     op = operator.ge
 129                 elif c_op == "<":
 130                     op = operator.lt
 131                 elif c_op == "<=":
 132                     op = operator.le
 133                 elif c_op == "=":
 134                     op = operator.eq
 135                 dist_value = c_property.get("distance").get("value")
 136                 my_access_distance_constraint = access_dist.AccessDistance(
 137                     constraint_id, constraint_type, constraint_demands,
 138                     _comparison_operator=op, _threshold=dist_value,
 139                     _location=self.locations[location_id])
 140                 self.constraints[my_access_distance_constraint.name] = \
 141                     my_access_distance_constraint
 142             elif constraint_type == "distance_between_demands":
 143                 c_property = constraint_info.get("properties")
 144                 op = operator.le  # default operator
 145                 c_op = c_property.get("distance").get("operator")
 146                 if c_op == ">":
 147                     op = operator.gt
 148                 elif c_op == ">=":
 149                     op = operator.ge
 150                 elif c_op == "<":
 151                     op = operator.lt
 152                 elif c_op == "<=":
 153                     op = operator.le
 154                 elif c_op == "=":
 155                     op = operator.eq
 156                 dist_value = c_property.get("distance").get("value")
 157                 my_aic_distance_constraint = aic_dist.AICDistance(
 158                     constraint_id, constraint_type, constraint_demands,
 159                     _comparison_operator=op, _threshold=dist_value)
 160                 self.constraints[my_aic_distance_constraint.name] = \
 161                     my_aic_distance_constraint
 162             elif constraint_type == "inventory_group":
 163                 my_inventory_group_constraint = \
 164                     inventory_group.InventoryGroup(
 165                         constraint_id, constraint_type, constraint_demands)
 166                 self.constraints[my_inventory_group_constraint.name] = \
 167                     my_inventory_group_constraint
 168             elif constraint_type == "region_fit":
 169                 c_property = constraint_info.get("properties")
 170                 controller = c_property.get("controller")
 171                 request = c_property.get("request")
 172                 # inventory type is cloud for region_fit
 173                 inventory_type = "cloud"
 174                 my_service_constraint = service_constraint.Service(
 175                     constraint_id, constraint_type, constraint_demands,
 176                     _controller=controller, _request=request, _cost=None,
 177                     _inventory_type=inventory_type)
 178                 self.constraints[my_service_constraint.name] = \
 179                     my_service_constraint
 180             elif constraint_type == "instance_fit":
 181                 c_property = constraint_info.get("properties")
 182                 controller = c_property.get("controller")
 183                 request = c_property.get("request")
 184                 # inventory type is service for instance_fit
 185                 inventory_type = "service"
 186                 my_service_constraint = service_constraint.Service(
 187                     constraint_id, constraint_type, constraint_demands,
 188                     _controller=controller, _request=request, _cost=None,
 189                     _inventory_type=inventory_type)
 190                 self.constraints[my_service_constraint.name] = \
 191                     my_service_constraint
 192             elif constraint_type == "zone":
 193                 c_property = constraint_info.get("properties")
 194                 location_id = c_property.get("location")
 195                 qualifier = c_property.get("qualifier")
 196                 category = c_property.get("category")
 197                 location = self.locations[location_id] if location_id else None
 198                 my_zone_constraint = zone.Zone(
 199                     constraint_id, constraint_type, constraint_demands,
 200                     _qualifier=qualifier, _category=category,
 201                     _location=location)
 202                 self.constraints[my_zone_constraint.name] = my_zone_constraint
 203             elif constraint_type == "attribute":
 204                 c_property = constraint_info.get("properties")
 205                 my_attribute_constraint = \
 206                     attribute_constraint.Attribute(constraint_id,
 207                                                    constraint_type,
 208                                                    constraint_demands,
 209                                                    _properties=c_property)
 210                 self.constraints[my_attribute_constraint.name] = \
 211                     my_attribute_constraint
 212             elif constraint_type == "threshold":
 213                 c_property = constraint_info.get("properties")
 214                 my_threshold_constraint = \
 215                     threshold.Threshold(constraint_id,
 216                                         constraint_type,
 217                                         constraint_demands,
 218                                         _properties=c_property)
 219                 self.constraints[my_threshold_constraint.name] = my_threshold_constraint
 220             elif constraint_type == "hpa":
 221                 LOG.debug("Creating constraint - {}".format(constraint_type))
 222                 c_property = constraint_info.get("properties")
 223                 my_hpa_constraint = hpa.HPA(constraint_id,
 224                                             constraint_type,
 225                                             constraint_demands,
 226                                             _properties=c_property)
 227                 self.constraints[my_hpa_constraint.name] = my_hpa_constraint
 228             elif constraint_type == "vim_fit":
 229                 LOG.debug("Creating constraint - {}".format(constraint_type))
 230                 c_property = constraint_info.get("properties")
 231                 my_vim_constraint = vim_fit.VimFit(constraint_id,
 232                                                    constraint_type,
 233                                                    constraint_demands,
 234                                                    _properties=c_property)
 235                 self.constraints[my_vim_constraint.name] = my_vim_constraint
 236             else:
 237                 LOG.error("unknown constraint type {}".format(constraint_type))
 238                 return
 239
 240         # get objective function
 241         if "objective" not in json_template["conductor_solver"] \
 242                 or not json_template["conductor_solver"]["objective"]:
 243             self.objective = objective.Objective()
 244         else:
 245             input_objective = json_template["conductor_solver"]["objective"]
 246             self.objective = objective.Objective()
 247             self.objective.goal = input_objective["goal"]
 248             self.objective.operation = input_objective["operation"]
 249             self.latencyTriage = TriageLatency()
 250
 251             LOG.info("objective function params")
 252             for operand_data in input_objective["operands"]:
 253                 if operand_data["function"] == "latency_between":
 254                     self.obj_func_param.append(operand_data["function_param"][1])
 255             LOG.info("done - objective function params")
 256             for operand_data in input_objective["operands"]:
 257                 operand = objective.Operand()
 258                 operand.operation = operand_data["operation"]
 259                 operand.weight = float(operand_data["weight"])
 260                 if operand_data["function"] == "latency_between":
 261                     LOG.debug("Processing objective function latency_between")
 262                     self.latencyTriage.takeOpimaztionType(operand_data["function"])
 263                     func = latency_between.LatencyBetween("latency_between")
 264                     func.region_group = self.assign_region_group_weight(country_groups, regions_maps)
 265                     param = operand_data["function_param"][0]
 266                     if param in self.locations:
 267                         func.loc_a = self.locations[param]
 268                     elif param in self.demands:
 269                         func.loc_a = self.demands[param]
 270                     param = operand_data["function_param"][1]
 271                     if param in self.locations:
 272                         func.loc_z = self.locations[param]
 273                     elif param in self.demands:
 274                         func.loc_z = self.demands[param]
 275                     operand.function = func
 276                 elif operand_data["function"] == "distance_between":
 277                     LOG.debug("Processing objective function distance_between")
 278                     self.latencyTriage.takeOpimaztionType(operand_data["function"])
 279                     func = distance_between.DistanceBetween("distance_between")
 280                     param = operand_data["function_param"][0]
 281                     if param in self.locations:
 282                         func.loc_a = self.locations[param]
 283                     elif param in self.demands:
 284                         func.loc_a = self.demands[param]
 285                     param = operand_data["function_param"][1]
 286                     if param in self.locations:
 287                         func.loc_z = self.locations[param]
 288                     elif param in self.demands:
 289                         func.loc_z = self.demands[param]
 290                     operand.function = func
 291                 elif operand_data["function"] == "aic_version":
 292                     self.objective.goal = "min_aic"
 293                     func = aic_version.AICVersion("aic_version")
 294                     func.loc = operand_data["function_param"]
 295                     operand.function = func
 296                 elif operand_data["function"] == "cost":
 297                     func = cost.Cost("cost")
 298                     func.loc = operand_data["function_param"]
 299                     operand.function = func
 300                 elif operand_data["function"] == "hpa_score":
 301                     func = hpa_score.HPAScore("hpa_score")
 302                     operand.function = func
 303
 304                 self.objective.operand_list.append(operand)
 305             self.latencyTriage.updateTriageLatencyDB(self.plan_id, self.request_id)
 306
 307     def assign_region_group_weight(self, countries, regions):
 308         """ assign the latency group value to the country and returns a map"""
 309         LOG.info("Processing Assigning Latency Weight to Countries ")
 310
 311         countries = self.resolve_countries(countries, regions,
 312                                            self.get_candidate_country_list())  # resolve the countries based on region type
 313         region_latency_weight = collections.OrderedDict()
 314         weight = 0
 315
 316         if countries is None:
 317             LOG.info("No countries available to assign latency weight ")
 318             return region_latency_weight
 319
 320         try:
 321             l_weight = ''
 322             for i, e in enumerate(countries):
 323                 if e is None: continue
 324                 for k, x in enumerate(e.split(',')):
 325                     region_latency_weight[x] = weight
 326                     l_weight += x + " : " + str(weight)
 327                     l_weight += ','
 328                 weight = weight + 1
 329             LOG.info("Latency Weights Assigned ")
 330             LOG.info(l_weight)
 331         except Exception as err:
 332             LOG.info("Exception while assigning the latency weights " + err)
 333             print(err)
 334         return region_latency_weight
 335
 336     def get_candidate_country_list(self):
 337         LOG.info("Processing Get Candidate Countries from demands  ")
 338         candidate_country_list = list()
 339         try:
 340
 341             candidate_countries = ''
 342             for demand_id, demands in self.demands.items():
 343                 candidate_countries += demand_id
 344                 for candidte in list(demands.resources.values()):   # Python 3 Conversion -- dict object to list object
 345                     candidate_country_list.append(candidte["country"])
 346                     candidate_countries += candidte["country"]
 347                     candidate_countries += ','
 348
 349             LOG.info("Available Candidate Countries " + candidate_countries)
 350         except Exception as err:
 351             print(err)
 352         return candidate_country_list
 353
 354     def resolve_countries(self, countries_list, regions_map, candidates_country_list):
 355         # check the map with given location and retrieve the values
 356         LOG.info("Resolving Countries ")
 357         if countries_list is None:
 358             LOG.info("No Countries are available ")
 359             return countries_list
 360
 361         countries_list = self.filter_invalid_rules(countries_list, regions_map)
 362
 363         if countries_list is not None and countries_list.__len__() > 0 and countries_list.__getitem__(
 364                 countries_list.__len__() - 1) == "*":
 365             self.process_wildcard_rules(candidates_country_list, countries_list)
 366         else:
 367             self.process_without_wildcard_rules(candidates_country_list, countries_list)
 368
 369         return countries_list
 370
 371     def process_without_wildcard_rules(self, candidates_country_list, countries_list):
 372         try:
 373             temp_country_list = list()
 374             for country_group in countries_list:
 375                 for country in country_group.split(','):
 376                     temp_country_list.append(country)
 377
 378             tmpcl = ''
 379             for cl in temp_country_list:
 380                 tmpcl += cl
 381                 tmpcl += ','
 382
 383             LOG.info("Countries List before diff " + tmpcl)
 384
 385             ccl = ''
 386             for cl in candidates_country_list:
 387                 ccl += cl
 388                 ccl += ','
 389
 390             LOG.info("Candidates Countries List before diff " + ccl)
 391
 392             # filterout the candidates that does not match the countries list
 393             # filter(lambda x: x not in countries_list, self.get_candidate_countries_list())
 394             LOG.info("Processing Difference between Candidate Countries and Latency Countries without *. ")
 395             diff_bw_candidates_and_countries = list(set(candidates_country_list).difference(temp_country_list))
 396             candcl = ''
 397             for cl in diff_bw_candidates_and_countries:
 398                 candcl += cl
 399                 candcl += ','
 400
 401             LOG.info("Available countries after processing diff between " + candcl)
 402
 403             self.drop_no_latency_rule_candidates(diff_bw_candidates_and_countries)
 404         except Exception as error:
 405             print(error)
 406
 407     def drop_no_latency_rule_candidates(self, diff_bw_candidates_and_countries):
 408
 409         cadidate_list_ = list()
 410         temp_candidates = dict()
 411
 412         for demand_id, demands in self.demands.items():
 413             LOG.info("demand id " + demand_id)
 414             for candidte in list(demands.resources.values()):    # Python 3 Conversion -- dict object to list object
 415                 LOG.info("candidate id " + candidte['candidate_id'])
 416                 dem_candidate = {demand_id: demands}
 417                 temp_candidates.update(dem_candidate)
 418
 419         droped_candidates = ''
 420         for demand_id, demands in temp_candidates.items():
 421             droped_candidates += demand_id
 422             for candidate in list(demands.resources.values()):   # Python 3 Conversion -- dict object to list object
 423                 if demand_id in self.obj_func_param and candidate["country"] in diff_bw_candidates_and_countries:
 424                     droped_candidates += candidate['candidate_id']
 425                     droped_candidates += ','
 426                     self.latencyTriage.latencyDroppedCandiate(candidate['candidate_id'], demand_id, reason="diff_bw_candidates_and_countries,Latecy weight ")
 427                     self.demands[demand_id].resources.pop(candidate['candidate_id'])
 428         LOG.info("dropped " + droped_candidates)
 429
 430         # for demand_id, candidate_list in self.demands:
 431         #     LOG.info("Candidates for demand " + demand_id)
 432         #     cadidate_list_ = self.demands[demand_id]['candidates']
 433         #     droped_candidates = ''
 434         #     xlen = cadidate_list_.__len__() - 1
 435         #     len = xlen
 436         #     # LOG.info("Candidate List Length "+str(len))
 437         #     for i in range(len + 1):
 438         #         # LOG.info("iteration " + i)
 439         #         LOG.info("Candidate Country " + cadidate_list_[xlen]["country"])
 440         #         if cadidate_list_[xlen]["country"] in diff_bw_candidates_and_countries:
 441         #             droped_candidates += cadidate_list_[xlen]["country"]
 442         #             droped_candidates += ','
 443         #             self.demands[demand_id]['candidates'].remove(cadidate_list_[xlen])
 444         #             # filter(lambda candidate: candidate in candidate_list["candidates"])
 445         #             # LOG.info("Droping Cadidate not eligible for latency weight. Candidate ID " + cadidate_list_[xlen]["candidate_id"] + " Candidate Country: "+cadidate_list_[xlen]["country"])
 446         #             xlen = xlen - 1
 447         #         if xlen < 0: break
 448         #     LOG.info("Dropped Candidate Countries " + droped_candidates + " from demand " + demand_id)
 449
 450     def process_wildcard_rules(self, candidates_country_list, countries_list, ):
 451         LOG.info("Processing the rules for " + countries_list.__getitem__(countries_list.__len__() - 1))
 452         candidate_countries = ''
 453         countries_list.remove(countries_list.__getitem__(
 454             countries_list.__len__() - 1))  # remove the wildcard and replace it with available candidates countries
 455         temp_country_list = list()
 456         for country_group in countries_list:
 457             for country in country_group.split(','):
 458                 temp_country_list.append(country)
 459         temp_countries = ''
 460         for cl in temp_country_list:
 461             temp_countries += cl
 462             temp_countries += ','
 463         LOG.info("Countries before diff " + temp_countries)
 464         ccl = ''
 465         for cl in candidates_country_list:
 466             ccl += cl
 467             ccl += ','
 468         LOG.info("Candidates Countries List before diff " + ccl)
 469         diff_bw_candidates_and_countries = list(set(candidates_country_list).difference(temp_country_list))
 470         LOG.info("Processing Difference between Candidate Countries and Latency Countries for * . ")
 471         for c_group in diff_bw_candidates_and_countries:
 472             candidate_countries += c_group
 473             candidate_countries += ','
 474         LOG.info("Difference: " + candidate_countries[:-1])
 475         if candidate_countries.__len__() > 0:
 476             LOG.info(candidate_countries[:-1])
 477             countries_list.append(candidate_countries[:-1])  # append the list of difference to existing countries
 478         ac = ''
 479         for cl in countries_list:
 480             ac += cl
 481             ac += ','
 482         LOG.info("Available countries after processing diff between " + ac)
 483
 484     def filter_invalid_rules(self, countries_list, regions_map):
 485         invalid_rules = list();
 486         for i, e in enumerate(countries_list):
 487             if e is None: continue
 488
 489             for k, region in enumerate(e.split(',')):
 490                 LOG.info("Processing the Rule for  " + region)
 491                 if region.__len__() != 3:
 492                     if region == "*":
 493                         continue
 494                     region_list = regions_map.get(region)
 495
 496                     if region_list is None:
 497                         LOG.info("Invalid region " + region)
 498                         invalid_rules.append(region)
 499                         continue
 500                     countries_list.remove(countries_list[i])
 501                     countries_list.insert(i, region_list)
 502         for ir in invalid_rules:
 503             LOG.info("Filtering out invalid rules from countries list ")
 504             LOG.info("invalid rule " + ir)
 505
 506         countries_list = list(filter(lambda country: (country not in invalid_rules), countries_list))
 507
 508         available_countries = ''
 509         for cl in countries_list:
 510             available_countries += cl
 511             available_countries += ','
 512
 513         LOG.info("Available countries after the filteration " + available_countries[:-1])
 514
 515         return countries_list
 516
 517     def reorder_constraint(self):
 518         # added manual ranking to the constraint type for optimizing purpose the last 2 are costly interaction
 519         for constraint_name, constraint in self.constraints.items():
 520             if constraint.constraint_type == "distance_to_location":
 521                 constraint.rank = 1
 522             elif constraint.constraint_type == "zone":
 523                 constraint.rank = 2
 524             elif constraint.constraint_type == "attribute":
 525                 constraint.rank = 3
 526             elif constraint.constraint_type == "hpa":
 527                 constraint.rank = 4
 528             elif constraint.constraint_type == "inventory_group":
 529                 constraint.rank = 5
 530             elif constraint.constraint_type == "vim_fit":
 531                 constraint.rank = 6
 532             elif constraint.constraint_type == "instance_fit":
 533                 constraint.rank = 7
 534             elif constraint.constraint_type == "region_fit":
 535                 constraint.rank = 8
 536             elif constraint.constraint_type == "threshold":
 537                 constraint.rank = 9
 538             else:
 539                 constraint.rank = 10
 540
 541     def attr_sort(self, attrs=['rank']):
 542         # this helper for sorting the rank
 543         return lambda k: [getattr(k, attr) for attr in attrs]
 544
 545     def sort_constraint_by_rank(self):
 546         # this sorts as rank
 547         for d_name, cl in self.demands.items():
 548             cl_list = cl.constraint_list
 549             cl_list.sort(key=self.attr_sort(attrs=['rank']))
 550
 551     def assgin_constraints_to_demands(self):
 552         # spread the constraints over the demands
 553         self.reorder_constraint()
 554         for constraint_name, constraint in self.constraints.items():
 555             for d in constraint.demand_list:
 556                 if d in list(self.demands.keys()):     # Python 3 Conversion -- dict object to list object
 557                     self.demands[d].constraint_list.append(constraint)
 558         self.sort_constraint_by_rank()