Digestion to ONAP
[optf/has.git] / conductor / conductor / solver / request / parser.py
1 #!/usr/bin/env python
2 #
3 # -------------------------------------------------------------------------
4 #   Copyright (c) 2015-2017 AT&T Intellectual Property
5 #
6 #   Licensed under the Apache License, Version 2.0 (the "License");
7 #   you may not use this file except in compliance with the License.
8 #   You may obtain a copy of the License at
9 #
10 #       http://www.apache.org/licenses/LICENSE-2.0
11 #
12 #   Unless required by applicable law or agreed to in writing, software
13 #   distributed under the License is distributed on an "AS IS" BASIS,
14 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 #   See the License for the specific language governing permissions and
16 #   limitations under the License.
17 #
18 # -------------------------------------------------------------------------
19 #
20
21
22 # import json
23 import collections
24 import operator
25 import random
26
27 from conductor.solver.optimizer.constraints \
28     import access_distance as access_dist
29 from conductor.solver.optimizer.constraints \
30     import aic_distance as aic_dist
31 from conductor.solver.optimizer.constraints \
32     import attribute as attribute_constraint
33 from conductor.solver.optimizer.constraints import hpa
34 from conductor.solver.optimizer.constraints \
35     import inventory_group
36 from conductor.solver.optimizer.constraints \
37     import service as service_constraint
38 from conductor.solver.optimizer.constraints import vim_fit
39 from conductor.solver.optimizer.constraints import zone
40 from conductor.solver.request import demand
41 from conductor.solver.request import objective
42 from conductor.solver.request.functions import aic_version
43 from conductor.solver.request.functions import cost
44 from conductor.solver.request.functions import distance_between
45 from conductor.solver.request.functions import hpa_score
46 from conductor.solver.request.functions import latency_between
47 from conductor.solver.request import objective
48 from conductor.solver.triage_tool.traige_latency import TriageLatency
49 from oslo_log import log
50
51 LOG = log.getLogger(__name__)
52
53
54 # FIXME(snarayanan): This is really a SolverRequest (or Request) object
55 class Parser(object):
56
57     demands = None  # type: Dict[Any, Any]
58     locations = None  # type: Dict[Any, Any]
59     obj_func_param = None
60
61     def __init__(self, _region_gen=None):
62         self.demands = {}
63         self.locations = {}
64         self.region_gen = _region_gen
65         self.constraints = {}
66         self.objective = None
67         self.obj_func_param = list()
68         self.cei = None
69         self.request_id = None
70         self.request_type = None
71         self.region_group = None
72
73     # def get_data_engine_interface(self):
74     #    self.cei = cei.ConstraintEngineInterface()
75
76     # FIXME(snarayanan): This should just be parse_template
77     def parse_template(self, json_template=None, country_groups=None, regions_maps=None):
78         if json_template is None:
79             LOG.error("No template specified")
80             return "Error"
81         # fd = open(self.region_gen.data_path + \
82         #     "/../dhv/dhv_test_template.json", "r")
83         # fd = open(template, "r")
84         # parse_template = json.load(fd)
85         # fd.close()
86
87         # get request type
88         self.request_type = json_template['conductor_solver']['request_type']
89
90         # get demands
91         demand_list = json_template["conductor_solver"]["demands"]
92         for demand_id, candidate_list in demand_list.items():
93             current_demand = demand.Demand(demand_id)
94             # candidate should only have minimal information like location_id
95             for candidate in candidate_list["candidates"]:
96                 candidate_id = candidate["candidate_id"]
97                 current_demand.resources[candidate_id] = candidate
98             current_demand.sort_base = 0  # this is only for testing
99             self.demands[demand_id] = current_demand
100
101         # get locations
102         location_list = json_template["conductor_solver"]["locations"]
103         for location_id, location_info in location_list.items():
104             loc = demand.Location(location_id)
105             loc.loc_type = "coordinates"
106             loc.value = (float(location_info["latitude"]),
107                          float(location_info["longitude"]))
108             loc.country = location_info[
109                 'country'] if 'country' in location_info else None
110             self.locations[location_id] = loc
111
112         # get constraints
113         input_constraints = json_template["conductor_solver"]["constraints"]
114         for constraint_id, constraint_info in input_constraints.items():
115             constraint_type = constraint_info["type"]
116             constraint_demands = list()
117             parsed_demands = constraint_info["demands"]
118             if isinstance(parsed_demands, list):
119                 for d in parsed_demands:
120                     constraint_demands.append(d)
121             else:
122                 constraint_demands.append(parsed_demands)
123             if constraint_type == "distance_to_location":
124                 c_property = constraint_info.get("properties")
125                 location_id = c_property.get("location")
126                 op = operator.le  # default operator
127                 c_op = c_property.get("distance").get("operator")
128                 if c_op == ">":
129                     op = operator.gt
130                 elif c_op == ">=":
131                     op = operator.ge
132                 elif c_op == "<":
133                     op = operator.lt
134                 elif c_op == "<=":
135                     op = operator.le
136                 elif c_op == "=":
137                     op = operator.eq
138                 dist_value = c_property.get("distance").get("value")
139                 my_access_distance_constraint = access_dist.AccessDistance(
140                     constraint_id, constraint_type, constraint_demands,
141                     _comparison_operator=op, _threshold=dist_value,
142                     _location=self.locations[location_id])
143                 self.constraints[my_access_distance_constraint.name] = \
144                     my_access_distance_constraint
145             elif constraint_type == "distance_between_demands":
146                 c_property = constraint_info.get("properties")
147                 op = operator.le  # default operator
148                 c_op = c_property.get("distance").get("operator")
149                 if c_op == ">":
150                     op = operator.gt
151                 elif c_op == ">=":
152                     op = operator.ge
153                 elif c_op == "<":
154                     op = operator.lt
155                 elif c_op == "<=":
156                     op = operator.le
157                 elif c_op == "=":
158                     op = operator.eq
159                 dist_value = c_property.get("distance").get("value")
160                 my_aic_distance_constraint = aic_dist.AICDistance(
161                     constraint_id, constraint_type, constraint_demands,
162                     _comparison_operator=op, _threshold=dist_value)
163                 self.constraints[my_aic_distance_constraint.name] = \
164                     my_aic_distance_constraint
165             elif constraint_type == "inventory_group":
166                 my_inventory_group_constraint = \
167                     inventory_group.InventoryGroup(
168                         constraint_id, constraint_type, constraint_demands)
169                 self.constraints[my_inventory_group_constraint.name] = \
170                     my_inventory_group_constraint
171             elif constraint_type == "region_fit":
172                 c_property = constraint_info.get("properties")
173                 controller = c_property.get("controller")
174                 request = c_property.get("request")
175                 # inventory type is cloud for region_fit
176                 inventory_type = "cloud"
177                 my_service_constraint = service_constraint.Service(
178                     constraint_id, constraint_type, constraint_demands,
179                     _controller=controller, _request=request, _cost=None,
180                     _inventory_type=inventory_type)
181                 self.constraints[my_service_constraint.name] = \
182                     my_service_constraint
183             elif constraint_type == "instance_fit":
184                 c_property = constraint_info.get("properties")
185                 controller = c_property.get("controller")
186                 request = c_property.get("request")
187                 # inventory type is service for instance_fit
188                 inventory_type = "service"
189                 my_service_constraint = service_constraint.Service(
190                     constraint_id, constraint_type, constraint_demands,
191                     _controller=controller, _request=request, _cost=None,
192                     _inventory_type=inventory_type)
193                 self.constraints[my_service_constraint.name] = \
194                     my_service_constraint
195             elif constraint_type == "zone":
196                 c_property = constraint_info.get("properties")
197                 location_id = c_property.get("location")
198                 qualifier = c_property.get("qualifier")
199                 category = c_property.get("category")
200                 location = self.locations[location_id] if location_id else None
201                 my_zone_constraint = zone.Zone(
202                     constraint_id, constraint_type, constraint_demands,
203                     _qualifier=qualifier, _category=category,
204                     _location=location)
205                 self.constraints[my_zone_constraint.name] = my_zone_constraint
206             elif constraint_type == "attribute":
207                 c_property = constraint_info.get("properties")
208                 my_attribute_constraint = \
209                     attribute_constraint.Attribute(constraint_id,
210                                                    constraint_type,
211                                                    constraint_demands,
212                                                    _properties=c_property)
213                 self.constraints[my_attribute_constraint.name] = \
214                     my_attribute_constraint
215             elif constraint_type == "hpa":
216                 LOG.debug("Creating constraint - {}".format(constraint_type))
217                 c_property = constraint_info.get("properties")
218                 my_hpa_constraint = hpa.HPA(constraint_id,
219                                             constraint_type,
220                                             constraint_demands,
221                                             _properties=c_property)
222                 self.constraints[my_hpa_constraint.name] = my_hpa_constraint
223             elif constraint_type == "vim_fit":
224                 LOG.debug("Creating constraint - {}".format(constraint_type))
225                 c_property = constraint_info.get("properties")
226                 my_vim_constraint = vim_fit.VimFit(constraint_id,
227                                                    constraint_type,
228                                                    constraint_demands,
229                                                    _properties=c_property)
230                 self.constraints[my_vim_constraint.name] = my_vim_constraint
231             else:
232                 LOG.error("unknown constraint type {}".format(constraint_type))
233                 return
234
235         # get objective function
236         if "objective" not in json_template["conductor_solver"] \
237                 or not json_template["conductor_solver"]["objective"]:
238             self.objective = objective.Objective()
239         else:
240             input_objective = json_template["conductor_solver"]["objective"]
241             self.objective = objective.Objective()
242             self.objective.goal = input_objective["goal"]
243             self.objective.operation = input_objective["operation"]
244             self.latencyTriage = TriageLatency()
245
246             LOG.info("objective function params")
247             for operand_data in input_objective["operands"]:
248                 if operand_data["function"] == "latency_between":
249                     self.obj_func_param.append(operand_data["function_param"][1])
250             LOG.info("done - objective function params")
251             for operand_data in input_objective["operands"]:
252                 operand = objective.Operand()
253                 operand.operation = operand_data["operation"]
254                 operand.weight = float(operand_data["weight"])
255                 if operand_data["function"] == "latency_between":
256                     LOG.debug("Processing objective function latency_between")
257                     self.latencyTriage.takeOpimaztionType(operand_data["function"])
258                     func = latency_between.LatencyBetween("latency_between")
259                     func.region_group = self.assign_region_group_weight(country_groups, regions_maps)
260                     param = operand_data["function_param"][0]
261                     if param in self.locations:
262                         func.loc_a = self.locations[param]
263                     elif param in self.demands:
264                         func.loc_a = self.demands[param]
265                     param = operand_data["function_param"][1]
266                     if param in self.locations:
267                         func.loc_z = self.locations[param]
268                     elif param in self.demands:
269                         func.loc_z = self.demands[param]
270                     operand.function = func
271                 elif operand_data["function"] == "distance_between":
272                     LOG.debug("Processing objective function distance_between")
273                     self.latencyTriage.takeOpimaztionType(operand_data["function"])
274                     func = distance_between.DistanceBetween("distance_between")
275                     param = operand_data["function_param"][0]
276                     if param in self.locations:
277                         func.loc_a = self.locations[param]
278                     elif param in self.demands:
279                         func.loc_a = self.demands[param]
280                     param = operand_data["function_param"][1]
281                     if param in self.locations:
282                         func.loc_z = self.locations[param]
283                     elif param in self.demands:
284                         func.loc_z = self.demands[param]
285                     operand.function = func
286                 elif operand_data["function"] == "aic_version":
287                     self.objective.goal = "min_aic"
288                     func = aic_version.AICVersion("aic_version")
289                     func.loc = operand_data["function_param"]
290                     operand.function = func
291                 elif operand_data["function"] == "cost":
292                     func = cost.Cost("cost")
293                     func.loc = operand_data["function_param"]
294                     operand.function = func
295                 elif operand_data["function"] == "hpa_score":
296                     func = hpa_score.HPAScore("hpa_score")
297                     operand.function = func
298
299                 self.objective.operand_list.append(operand)
300             self.latencyTriage.updateTriageLatencyDB(self.plan_id, self.request_id)
301
302     def assign_region_group_weight(self, countries, regions):
303         """ assign the latency group value to the country and returns a map"""
304         LOG.info("Processing Assigning Latency Weight to Countries ")
305
306         countries = self.resolve_countries(countries, regions,
307                                            self.get_candidate_country_list())  # resolve the countries based on region type
308         region_latency_weight = collections.OrderedDict()
309         weight = 0
310
311         if countries is None:
312             LOG.info("No countries available to assign latency weight ")
313             return region_latency_weight
314
315         try:
316             l_weight = ''
317             for i, e in enumerate(countries):
318                 if e is None: continue
319                 for k, x in enumerate(e.split(',')):
320                     region_latency_weight[x] = weight
321                     l_weight += x + " : " + str(weight)
322                     l_weight += ','
323                 weight = weight + 1
324             LOG.info("Latency Weights Assigned ")
325             LOG.info(l_weight)
326         except Exception as err:
327             LOG.info("Exception while assigning the latency weights " + err)
328             print(err)
329         return region_latency_weight
330
331     def get_candidate_country_list(self):
332         LOG.info("Processing Get Candidate Countries from demands  ")
333         candidate_country_list = list()
334         try:
335
336             candidate_countries = ''
337             for demand_id, demands in self.demands.items():
338                 candidate_countries += demand_id
339                 for candidte in demands.resources.values():
340                     candidate_country_list.append(candidte["country"])
341                     candidate_countries += candidte["country"]
342                     candidate_countries += ','
343
344             LOG.info("Available Candidate Countries " + candidate_countries)
345         except Exception as err:
346             print(err)
347         return candidate_country_list
348
349     def resolve_countries(self, countries_list, regions_map, candidates_country_list):
350         # check the map with given location and retrieve the values
351         LOG.info("Resolving Countries ")
352         if countries_list is None:
353             LOG.info("No Countries are available ")
354             return countries_list
355
356         countries_list = self.filter_invalid_rules(countries_list, regions_map)
357
358         if countries_list is not None and countries_list.__len__() > 0 and countries_list.__getitem__(
359                 countries_list.__len__() - 1) == "*":
360             self.process_wildcard_rules(candidates_country_list, countries_list)
361         else:
362             self.process_without_wildcard_rules(candidates_country_list, countries_list)
363
364         return countries_list
365
366     def process_without_wildcard_rules(self, candidates_country_list, countries_list):
367         try:
368             temp_country_list = list()
369             for country_group in countries_list:
370                 for country in country_group.split(','):
371                     temp_country_list.append(country)
372
373             tmpcl = ''
374             for cl in temp_country_list:
375                 tmpcl += cl
376                 tmpcl += ','
377
378             LOG.info("Countries List before diff " + tmpcl)
379
380             ccl = ''
381             for cl in candidates_country_list:
382                 ccl += cl
383                 ccl += ','
384
385             LOG.info("Candidates Countries List before diff " + ccl)
386
387             # filterout the candidates that does not match the countries list
388             # filter(lambda x: x not in countries_list, self.get_candidate_countries_list())
389             LOG.info("Processing Difference between Candidate Countries and Latency Countries without *. ")
390             diff_bw_candidates_and_countries = list(set(candidates_country_list).difference(temp_country_list))
391             candcl = ''
392             for cl in diff_bw_candidates_and_countries:
393                 candcl += cl
394                 candcl += ','
395
396             LOG.info("Available countries after processing diff between " + candcl)
397
398             self.drop_no_latency_rule_candidates(diff_bw_candidates_and_countries)
399         except Exception as error:
400             print(error)
401
402     def drop_no_latency_rule_candidates(self, diff_bw_candidates_and_countries):
403
404         cadidate_list_ = list()
405         temp_candidates = dict()
406
407         for demand_id, demands in self.demands.items():
408             LOG.info("demand id " + demand_id)
409             for candidte in demands.resources.values():
410                 LOG.info("candidate id " + candidte['candidate_id'])
411                 dem_candidate = {demand_id: demands}
412                 temp_candidates.update(dem_candidate)
413
414         droped_candidates = ''
415         for demand_id, demands in temp_candidates.items():
416             droped_candidates += demand_id
417             for candidate in demands.resources.values():
418                 if demand_id in self.obj_func_param and candidate["country"] in diff_bw_candidates_and_countries:
419                     droped_candidates += candidate['candidate_id']
420                     droped_candidates += ','
421                     self.latencyTriage.latencyDroppedCandiate(candidate['candidate_id'], demand_id, reason="diff_bw_candidates_and_countries,Latecy weight ")
422                     self.demands[demand_id].resources.pop(candidate['candidate_id'])
423         LOG.info("dropped " + droped_candidates)
424
425         # for demand_id, candidate_list in self.demands:
426         #     LOG.info("Candidates for demand " + demand_id)
427         #     cadidate_list_ = self.demands[demand_id]['candidates']
428         #     droped_candidates = ''
429         #     xlen = cadidate_list_.__len__() - 1
430         #     len = xlen
431         #     # LOG.info("Candidate List Length "+str(len))
432         #     for i in range(len + 1):
433         #         # LOG.info("iteration " + i)
434         #         LOG.info("Candidate Country " + cadidate_list_[xlen]["country"])
435         #         if cadidate_list_[xlen]["country"] in diff_bw_candidates_and_countries:
436         #             droped_candidates += cadidate_list_[xlen]["country"]
437         #             droped_candidates += ','
438         #             self.demands[demand_id]['candidates'].remove(cadidate_list_[xlen])
439         #             # filter(lambda candidate: candidate in candidate_list["candidates"])
440         #             # LOG.info("Droping Cadidate not eligible for latency weight. Candidate ID " + cadidate_list_[xlen]["candidate_id"] + " Candidate Country: "+cadidate_list_[xlen]["country"])
441         #             xlen = xlen - 1
442         #         if xlen < 0: break
443         #     LOG.info("Dropped Candidate Countries " + droped_candidates + " from demand " + demand_id)
444
445     def process_wildcard_rules(self, candidates_country_list, countries_list, ):
446         LOG.info("Processing the rules for " + countries_list.__getitem__(countries_list.__len__() - 1))
447         candidate_countries = ''
448         countries_list.remove(countries_list.__getitem__(
449             countries_list.__len__() - 1))  # remove the wildcard and replace it with available candidates countries
450         temp_country_list = list()
451         for country_group in countries_list:
452             for country in country_group.split(','):
453                 temp_country_list.append(country)
454         temp_countries = ''
455         for cl in temp_country_list:
456             temp_countries += cl
457             temp_countries += ','
458         LOG.info("Countries before diff " + temp_countries)
459         ccl = ''
460         for cl in candidates_country_list:
461             ccl += cl
462             ccl += ','
463         LOG.info("Candidates Countries List before diff " + ccl)
464         diff_bw_candidates_and_countries = list(set(candidates_country_list).difference(temp_country_list))
465         LOG.info("Processing Difference between Candidate Countries and Latency Countries for * . ")
466         for c_group in diff_bw_candidates_and_countries:
467             candidate_countries += c_group
468             candidate_countries += ','
469         LOG.info("Difference: " + candidate_countries[:-1])
470         if candidate_countries.__len__() > 0:
471             LOG.info(candidate_countries[:-1])
472             countries_list.append(candidate_countries[:-1])  # append the list of difference to existing countries
473         ac = ''
474         for cl in countries_list:
475             ac += cl
476             ac += ','
477         LOG.info("Available countries after processing diff between " + ac)
478
479     def filter_invalid_rules(self, countries_list, regions_map):
480         invalid_rules = list();
481         for i, e in enumerate(countries_list):
482             if e is None: continue
483
484             for k, region in enumerate(e.split(',')):
485                 LOG.info("Processing the Rule for  " + region)
486                 if region.__len__() != 3:
487                     if region == "*":
488                         continue
489                     region_list = regions_map.get(region)
490
491                     if region_list is None:
492                         LOG.info("Invalid region " + region)
493                         invalid_rules.append(region)
494                         continue
495                     countries_list.remove(countries_list[i])
496                     countries_list.insert(i, region_list)
497         for ir in invalid_rules:
498             LOG.info("Filtering out invalid rules from countries list ")
499             LOG.info("invalid rule " + ir)
500
501         countries_list = list(filter(lambda country: (country not in invalid_rules), countries_list))
502
503         available_countries = ''
504         for cl in countries_list:
505             available_countries += cl
506             available_countries += ','
507
508         LOG.info("Available countries after the filteration " + available_countries[:-1])
509
510         return countries_list
511
512     def get_data_from_aai_simulator(self):
513         loc = demand.Location("uCPE")
514         loc.loc_type = "coordinates"
515         latitude = random.uniform(self.region_gen.least_latitude,
516                                   self.region_gen.most_latitude)
517         longitude = random.uniform(self.region_gen.least_longitude,
518                                    self.region_gen.most_longitude)
519         loc.value = (latitude, longitude)
520         self.locations[loc.name] = loc
521
522         demand1 = demand.Demand("vVIG")
523         demand1.resources = self.region_gen.regions
524         demand1.sort_base = 0  # this is only for testing
525         self.demands[demand1.name] = demand1
526
527         demand2 = demand.Demand("vGW")
528         demand2.resources = self.region_gen.regions
529         demand2.sort_base = 2  # this is only for testing
530         self.demands[demand2.name] = demand2
531
532         demand3 = demand.Demand("vVIG2")
533         demand3.resources = self.region_gen.regions
534         demand3.sort_base = 1  # this is only for testing
535         self.demands[demand3.name] = demand3
536
537         demand4 = demand.Demand("vGW2")
538         demand4.resources = self.region_gen.regions
539         demand4.sort_base = 3  # this is only for testing
540         self.demands[demand4.name] = demand4
541
542         constraint_list = []
543
544         access_distance = access_dist.AccessDistance(
545             "uCPE-all", "access_distance",
546             [demand1.name, demand2.name, demand3.name, demand4.name],
547             _comparison_operator=operator.le, _threshold=50000,
548             _location=loc)
549         constraint_list.append(access_distance)
550
551         '''
552         access_distance = access_dist.AccessDistance(
553             "uCPE-all", "access_distance", [demand1.name, demand2.name],
554             _comparison_operator=operator.le, _threshold=5000, _location=loc)
555         constraint_list.append(access_distance)
556
557         aic_distance = aic_dist.AICDistance(
558             "vVIG-vGW", "aic_distance", [demand1.name, demand2.name],
559             _comparison_operator=operator.le, _threshold=50)
560         constraint_list.append(aic_distance)
561
562         same_zone = zone.Zone(
563             "same-zone", "zone", [demand1.name, demand2.name],
564             _qualifier="same", _category="zone1")
565         constraint_list.append(same_zone)
566         '''
567
568     def reorder_constraint(self):
569         # added manual ranking to the constraint type for optimizing purpose the last 2 are costly interaction
570         for constraint_name, constraint in self.constraints.items():
571             if constraint.constraint_type == "distance_to_location":
572                 constraint.rank = 1
573             elif constraint.constraint_type == "zone":
574                 constraint.rank = 2
575             elif constraint.constraint_type == "attribute":
576                 constraint.rank = 3
577             elif constraint.constraint_type == "hpa":
578                 constraint.rank = 4
579             elif constraint.constraint_type == "inventory_group":
580                 constraint.rank = 5
581             elif constraint.constraint_type == "vim_fit":
582                 constraint.rank = 6
583             elif constraint.constraint_type == "instance_fit":
584                 constraint.rank = 7
585             elif constraint.constraint_type == "region_fit":
586                 constraint.rank = 8
587             else:
588                 constraint.rank = 9
589
590     def attr_sort(self, attrs=['rank']):
591         # this helper for sorting the rank
592         return lambda k: [getattr(k, attr) for attr in attrs]
593
594     def sort_constraint_by_rank(self):
595         # this sorts as rank
596         for d_name, cl in self.demands.items():
597             cl_list = cl.constraint_list
598             cl_list.sort(key=self.attr_sort(attrs=['rank']))
599
600     def assgin_constraints_to_demands(self):
601         # self.parse_dhv_template() # get data from DHV template
602         # self.get_data_from_aai_simulator() # get data from aai simulation
603         # renaming simulate to assgin_constraints_to_demands
604         # spread the constraints over the demands
605         self.reorder_constraint()
606         for constraint_name, constraint in self.constraints.items():
607             for d in constraint.demand_list:
608                 if d in self.demands.keys():
609                     self.demands[d].constraint_list.append(constraint)
610         self.sort_constraint_by_rank()