conductor/conductor/controller/translator.py

   1 #
   2 # -------------------------------------------------------------------------
   3 #   Copyright (c) 2015-2017 AT&T Intellectual Property
   4 #   Copyright (C) 2020 Wipro Limited.
   5 #
   6 #   Licensed under the Apache License, Version 2.0 (the "License");
   7 #   you may not use this file except in compliance with the License.
   8 #   You may obtain a copy of the License at
   9 #
  10 #       http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #   Unless required by applicable law or agreed to in writing, software
  13 #   distributed under the License is distributed on an "AS IS" BASIS,
  14 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 #   See the License for the specific language governing permissions and
  16 #   limitations under the License.
  17 #
  18 # -------------------------------------------------------------------------
  19 #
  20
  21 import copy
  22 import datetime
  23 import json
  24 import os
  25 import uuid
  26
  27 import six
  28 import yaml
  29 from conductor import __file__ as conductor_root
  30 from conductor import messaging
  31 from conductor import service
  32
  33 from conductor.common import threshold
  34 from conductor.common.music import messaging as music_messaging
  35 from conductor.data.plugins.triage_translator.triage_translator_data import TraigeTranslatorData
  36 from conductor.data.plugins.triage_translator.triage_translator import TraigeTranslator
  37 from oslo_config import cfg
  38 from oslo_log import log
  39
  40 LOG = log.getLogger(__name__)
  41
  42 CONF = cfg.CONF
  43
  44 VERSIONS = ["2016-11-01", "2017-10-10", "2018-02-01"]
  45 LOCATION_KEYS = ['latitude', 'longitude', 'host_name', 'clli_code']
  46 INVENTORY_PROVIDERS = ['aai']
  47 INVENTORY_TYPES = ['cloud', 'service', 'transport', 'vfmodule', 'nssi']
  48 DEFAULT_INVENTORY_PROVIDER = INVENTORY_PROVIDERS[0]
  49 CANDIDATE_KEYS = ['candidate_id', 'cost', 'inventory_type', 'location_id',
  50                   'location_type']
  51 DEMAND_KEYS = ['filtering_attributes', 'passthrough_attributes', 'candidates', 'complex', 'conflict_identifier',
  52                'customer_id', 'default_cost', 'excluded_candidates',
  53                'existing_placement', 'flavor', 'inventory_provider',
  54                'inventory_type', 'port_key', 'region', 'required_candidates',
  55                'service_id', 'service_resource_id', 'service_subscription',
  56                'service_type', 'subdivision', 'unique', 'vlan_key']
  57 CONSTRAINT_KEYS = ['type', 'demands', 'properties']
  58 CONSTRAINTS = {
  59     # constraint_type: {
  60     #   split: split into individual constraints, one per demand
  61     #   required: list of required property names,
  62     #   optional: list of optional property names,
  63     #   thresholds: dict of property/base-unit pairs for threshold parsing
  64     #   allowed: dict of keys and allowed values (if controlled vocab);
  65     #            only use this for Conductor-controlled values!
  66     # }
  67     'attribute': {
  68         'split': True,
  69         'required': ['evaluate'],
  70     },
  71     'threshold': {
  72         'split': True,
  73         'required': ['evaluate'],
  74     },
  75     'distance_between_demands': {
  76         'required': ['distance'],
  77         'thresholds': {
  78             'distance': 'distance'
  79         },
  80     },
  81     'distance_to_location': {
  82         'split': True,
  83         'required': ['distance', 'location'],
  84         'thresholds': {
  85             'distance': 'distance'
  86         },
  87     },
  88     'instance_fit': {
  89         'split': True,
  90         'required': ['controller'],
  91         'optional': ['request'],
  92     },
  93     'inventory_group': {},
  94     'region_fit': {
  95         'split': True,
  96         'required': ['controller'],
  97         'optional': ['request'],
  98     },
  99     'zone': {
 100         'required': ['qualifier', 'category'],
 101         'optional': ['location'],
 102         'allowed': {'qualifier': ['same', 'different'],
 103                     'category': ['disaster', 'region', 'complex', 'country',
 104                                  'time', 'maintenance']},
 105     },
 106     'vim_fit': {
 107         'split': True,
 108         'required': ['controller'],
 109         'optional': ['request'],
 110     },
 111     'hpa': {
 112         'split': True,
 113         'required': ['evaluate'],
 114     },
 115 }
 116 HPA_FEATURES = ['architecture', 'hpa-feature', 'hpa-feature-attributes',
 117                 'hpa-version', 'mandatory', 'directives']
 118 HPA_OPTIONAL = ['score']
 119 HPA_ATTRIBUTES = ['hpa-attribute-key', 'hpa-attribute-value', 'operator']
 120 HPA_ATTRIBUTES_OPTIONAL = ['unit']
 121
 122
 123 class TranslatorException(Exception):
 124     pass
 125
 126
 127 class Translator(object):
 128     """Template translator.
 129
 130     Takes an input template and translates it into
 131     something the solver can use. Calls the data service
 132     as needed, giving it the inventory provider as context.
 133     Presently the only inventory provider is A&AI. Others
 134     may be added in the future.
 135     """
 136
 137     def __init__(self, conf, plan_name, plan_id, template):
 138         self.conf = conf
 139         self._template = copy.deepcopy(template)
 140         self._plan_name = plan_name
 141         self._plan_id = plan_id
 142         self._translation = None
 143         self._valid = False
 144         self._ok = False
 145         self.triageTranslatorData= TraigeTranslatorData()
 146         self.triageTranslator = TraigeTranslator()
 147         # Set up the RPC service(s) we want to talk to.
 148         self.data_service = self.setup_rpc(self.conf, "data")
 149
 150     def setup_rpc(self, conf, topic):
 151         """Set up the RPC Client"""
 152         # TODO(jdandrea): Put this pattern inside music_messaging?
 153         transport = messaging.get_transport(conf=conf)
 154         target = music_messaging.Target(topic=topic)
 155         client = music_messaging.RPCClient(conf=conf,
 156                                            transport=transport,
 157                                            target=target)
 158         return client
 159
 160     def create_components(self):
 161         # TODO(jdandrea): Make deep copies so the template is untouched
 162         self._version = self._template.get("homing_template_version")
 163         self._parameters = self._template.get("parameters", {})
 164         self._locations = self._template.get("locations", {})
 165         self._demands = self._template.get("demands", {})
 166         self._constraints = self._template.get("constraints", {})
 167         self._optmization = self._template.get("optimization", {})
 168         self._reservations = self._template.get("reservation", {})
 169
 170         if isinstance(self._version, datetime.date):
 171             self._version = str(self._version)
 172
 173     def validate_components(self):
 174         """Cursory validation of template components.
 175
 176         More detailed validation happens while parsing each component.
 177         """
 178         self._valid = False
 179
 180         # Check version
 181         if self._version not in VERSIONS:
 182             raise TranslatorException(
 183                 "conductor_template_version must be one "
 184                 "of: {}".format(', '.join(VERSIONS)))
 185
 186         # Check top level structure
 187         components = {
 188             "parameters": {
 189                 "name": "Parameter",
 190                 "content": self._parameters,
 191             },
 192             "locations": {
 193                 "name": "Location",
 194                 "keys": LOCATION_KEYS,
 195                 "content": self._locations,
 196             },
 197             "demands": {
 198                 "name": "Demand",
 199                 "content": self._demands,
 200             },
 201             "constraints": {
 202                 "name": "Constraint",
 203                 "keys": CONSTRAINT_KEYS,
 204                 "content": self._constraints,
 205             },
 206             "optimization": {
 207                 "name": "Optimization",
 208                 "content": self._optmization,
 209             },
 210             "reservations": {
 211                 "name": "Reservation",
 212                 "content": self._reservations,
 213             }
 214         }
 215         for name, component in components.items():
 216             name = component.get('name')
 217             keys = component.get('keys', None)
 218             content = component.get('content')
 219
 220             if type(content) is not dict:
 221                 raise TranslatorException(
 222                     "{} section must be a dictionary".format(name))
 223             for content_name, content_def in content.items():
 224                 if not keys:
 225                     continue
 226
 227                 for key in content_def:
 228                     if key not in keys:
 229                         raise TranslatorException(
 230                             "{} {} has an invalid key {}".format(
 231                                 name, content_name, key))
 232
 233         demand_keys = list(self._demands.keys())     # Python 3 Conversion -- dict object to list object
 234         location_keys = list(self._locations.keys())     # Python 3 Conversion -- dict object to list object
 235         for constraint_name, constraint in self._constraints.items():
 236
 237             # Require a single demand (string), or a list of one or more.
 238             demands = constraint.get('demands')
 239             if isinstance(demands, six.string_types):
 240                 demands = [demands]
 241             if not isinstance(demands, list) or len(demands) < 1:
 242                 raise TranslatorException(
 243                     "Demand list for Constraint {} must be "
 244                     "a list of names or a string with one name".format(
 245                         constraint_name))
 246             if not set(demands).issubset(demand_keys + location_keys):
 247                 raise TranslatorException(
 248                     "Undefined Demand(s) {} in Constraint '{}'".format(
 249                         list(set(demands).difference(demand_keys)),
 250                         constraint_name))
 251
 252             properties = constraint.get('properties', None)
 253             if properties:
 254                 location = properties.get('location', None)
 255                 if location:
 256                     if location not in location_keys:
 257                         raise TranslatorException(
 258                             "Location {} in Constraint {} is undefined".format(
 259                                 location, constraint_name))
 260
 261         self._valid = True
 262
 263     def _parse_parameters(self, obj, path=[]):
 264         """Recursively parse all {get_param: X} occurrences
 265
 266         This modifies obj in-place. If you want to keep the original,
 267         pass in a deep copy.
 268         """
 269         # Ok to start with a string ...
 270         if isinstance(path, six.string_types):
 271             # ... but the breadcrumb trail goes in an array.
 272             path = [path]
 273
 274         # Traverse a list
 275         if type(obj) is list:
 276             for idx, val in enumerate(obj, start=0):
 277                 # Add path to the breadcrumb trail
 278                 new_path = list(path)
 279                 new_path[-1] += "[{}]".format(idx)
 280
 281                 # Look at each element.
 282                 obj[idx] = self._parse_parameters(val, new_path)
 283
 284         # Traverse a dict
 285         elif type(obj) is dict:
 286             # Did we find a "{get_param: ...}" intrinsic?
 287             if list(obj.keys()) == ['get_param']:
 288                 param_name = obj['get_param']
 289
 290                 # The parameter name must be a string.
 291                 if not isinstance(param_name, six.string_types):
 292                     path_str = ' > '.join(path)
 293                     raise TranslatorException(
 294                         "Parameter name '{}' not a string in path {}".format(
 295                             param_name, path_str))
 296
 297                 # Parameter name must be defined.
 298                 if param_name not in self._parameters:
 299                     path_str = ' > '.join(path)
 300                     raise TranslatorException(
 301                         "Parameter '{}' undefined in path {}".format(
 302                             param_name, path_str))
 303
 304                 # Return the value in place of the call.
 305                 return self._parameters.get(param_name)
 306
 307             # Not an intrinsic. Traverse as usual.
 308             for key in list(obj.keys()):
 309                 # Add path to the breadcrumb trail.
 310                 new_path = list(path)
 311                 new_path.append(key)
 312
 313                 # Look at each key/value pair.
 314                 obj[key] = self._parse_parameters(obj[key], new_path)
 315
 316         # Return whatever we have after unwinding.
 317         return obj
 318
 319     def parse_parameters(self):
 320         """Resolve all parameters references."""
 321         locations = copy.deepcopy(self._locations)
 322         self._locations = self._parse_parameters(locations, 'locations')
 323
 324         demands = copy.deepcopy(self._demands)
 325         self._demands = self._parse_parameters(demands, 'demands')
 326
 327         constraints = copy.deepcopy(self._constraints)
 328         self._constraints = self._parse_parameters(constraints, 'constraints')
 329
 330         reservations = copy.deepcopy(self._reservations)
 331         self._reservations = self._parse_parameters(reservations,
 332                                                     'reservations')
 333
 334     def parse_locations(self, locations):
 335         """Prepare the locations for use by the solver."""
 336         parsed = {}
 337         for location, args in locations.items():
 338             ctxt = {
 339                 'plan_id': self._plan_id,
 340                 'keyspace': self.conf.keyspace
 341             }
 342
 343             latitude = args.get("latitude")
 344             longitude = args.get("longitude")
 345
 346             if latitude and longitude:
 347                 resolved_location = {"latitude": latitude, "longitude": longitude}
 348             else:
 349                 # ctxt = {}
 350                 response = self.data_service.call(
 351                     ctxt=ctxt,
 352                     method="resolve_location",
 353                     args=args)
 354
 355                 resolved_location = \
 356                     response and response.get('resolved_location')
 357             if not resolved_location:
 358                 raise TranslatorException(
 359                     "Unable to resolve location {}".format(location)
 360                 )
 361             parsed[location] = resolved_location
 362         return parsed
 363
 364     def parse_demands(self, demands):
 365         """Validate/prepare demands for use by the solver."""
 366         if type(demands) is not dict:
 367             raise TranslatorException("Demands must be provided in "
 368                                       "dictionary form")
 369
 370         # Look at each demand
 371         demands_copy = copy.deepcopy(demands)
 372         parsed = {}
 373         for name, requirements in demands_copy.items():
 374             inventory_candidates = []
 375             for requirement in requirements:
 376                 for key in requirement:
 377                     if key not in DEMAND_KEYS:
 378                         raise TranslatorException(
 379                             "Demand {} has an invalid key {}".format(
 380                                 requirement, key))
 381
 382                 if 'candidates' in requirement:
 383                     # Candidates *must* specify an inventory provider
 384                     provider = requirement.get("inventory_provider")
 385                     if provider and provider not in INVENTORY_PROVIDERS:
 386                         raise TranslatorException(
 387                             "Unsupported inventory provider {} "
 388                             "in demand {}".format(provider, name))
 389                     else:
 390                         provider = DEFAULT_INVENTORY_PROVIDER
 391
 392                     # Check each candidate
 393                     for candidate in requirement.get('candidates'):
 394                         # Must be a dictionary
 395                         if type(candidate) is not dict:
 396                             raise TranslatorException(
 397                                 "Candidate found in demand {} that is "
 398                                 "not a dictionary".format(name))
 399
 400                         # Must have only supported keys
 401                         for key in list(candidate.keys()):
 402                             if key not in CANDIDATE_KEYS:
 403                                 raise TranslatorException(
 404                                     "Candidate with invalid key {} found "
 405                                     "in demand {}".format(key, name)
 406                                 )
 407
 408                         # TODO(jdandrea): Check required/optional keys
 409
 410                         # Set the inventory provider if not already
 411                         candidate['inventory_provider'] = \
 412                             candidate.get('inventory_provider', provider)
 413
 414                         # Set cost if not already (default cost is 0?)
 415                         candidate['cost'] = candidate.get('cost', 0)
 416
 417                         # Add to our list of parsed candidates
 418                         inventory_candidates.append(candidate)
 419
 420                 # candidates are specified through inventory providers
 421                 # Do the basic sanity checks for inputs
 422                 else:
 423                     # inventory provider MUST be specified
 424                     provider = requirement.get("inventory_provider")
 425                     if not provider:
 426                         raise TranslatorException(
 427                             "Inventory provider not specified "
 428                             "in demand {}".format(name)
 429                         )
 430                     elif provider and provider not in INVENTORY_PROVIDERS:
 431                         raise TranslatorException(
 432                             "Unsupported inventory provider {} "
 433                             "in demand {}".format(provider, name)
 434                         )
 435                     else:
 436                         provider = DEFAULT_INVENTORY_PROVIDER
 437                         requirement['provider'] = provider
 438
 439                     # inventory type MUST be specified
 440                     inventory_type = requirement.get('inventory_type')
 441                     if not inventory_type or inventory_type == '':
 442                         raise TranslatorException(
 443                             "Inventory type not specified for "
 444                             "demand {}".format(name)
 445                         )
 446                     if inventory_type and \
 447                             inventory_type not in INVENTORY_TYPES:
 448                         raise TranslatorException(
 449                             "Unknown inventory type {} specified for "
 450                             "demand {}".format(inventory_type, name)
 451                         )
 452
 453                     # For service and vfmodule inventories, customer_id and
 454                     # service_type MUST be specified
 455                     if inventory_type == 'service' or inventory_type == 'vfmodule':
 456                         filtering_attributes = requirement.get('filtering_attributes')
 457
 458                         if filtering_attributes:
 459                             customer_id = filtering_attributes.get('customer-id')
 460                             global_customer_id = filtering_attributes.get('global-customer-id')
 461                             if global_customer_id:
 462                                 customer_id = global_customer_id
 463                         else:
 464                             # for backward compatibility
 465                             customer_id = requirement.get('customer_id')
 466                             service_type = requirement.get('service_type')
 467
 468                         if not customer_id:
 469                             raise TranslatorException(
 470                                 "Customer ID not specified for "
 471                                 "demand {}".format(name)
 472                             )
 473                         if not filtering_attributes and not service_type:
 474                             raise TranslatorException(
 475                                 "Service Type not specified for "
 476                                 "demand {}".format(name)
 477                             )
 478
 479                 # TODO(jdandrea): Check required/optional keys for requirement
 480                 # elif 'inventory_type' in requirement:
 481                 #     # For now this is just a stand-in candidate
 482                 #     candidate = {
 483                 #         'inventory_provider':
 484                 #             requirement.get('inventory_provider',
 485                 #                             DEFAULT_INVENTORY_PROVIDER),
 486                 #         'inventory_type':
 487                 #             requirement.get('inventory_type', ''),
 488                 #         'candidate_id': '',
 489                 #         'location_id': '',
 490                 #         'location_type': '',
 491                 #         'cost': 0,
 492                 #     }
 493                 #
 494                 #     # Add to our list of parsed candidates
 495                 #     inventory_candidates.append(candidate)
 496
 497             # Ask conductor-data for one or more candidates.
 498             ctxt = {
 499                 "plan_id": self._plan_id,
 500                 "plan_name": self._plan_name,
 501                 "keyspace": self.conf.keyspace,
 502             }
 503             args = {
 504                 "demands": {
 505                     name: requirements,
 506                 },
 507                 "plan_info":{
 508                     "plan_id": self._plan_id,
 509                     "plan_name": self._plan_name
 510                 },
 511                 "triage_translator_data": self.triageTranslatorData.__dict__
 512
 513             }
 514
 515             # Check if required_candidate and excluded candidate
 516             # are mutually exclusive.
 517             for requirement in requirements:
 518                 required_candidates = requirement.get("required_candidates")
 519                 excluded_candidates = requirement.get("excluded_candidates")
 520                 if (required_candidates and
 521                     excluded_candidates and
 522                     set(map(lambda entry: entry['candidate_id'],
 523                             required_candidates))
 524                     & set(map(lambda entry: entry['candidate_id'],
 525                               excluded_candidates))):
 526                     raise TranslatorException(
 527                         "Required candidate list and excluded candidate"
 528                         " list are not mutually exclusive for demand"
 529                         " {}".format(name)
 530                     )
 531             response = self.data_service.call(
 532                 ctxt=ctxt,
 533                 method="resolve_demands",
 534                 args=args)
 535
 536             resolved_demands = \
 537                 response and response.get('resolved_demands')
 538             triage_data_trans = \
 539                 response and response.get('trans')
 540
 541             required_candidates = resolved_demands \
 542                 .get('required_candidates')
 543             if not resolved_demands:
 544                 self.triageTranslator.thefinalCallTrans(triage_data_trans)
 545                 raise TranslatorException(
 546                     "Unable to resolve inventory "
 547                     "candidates for demand {}"
 548                     .format(name)
 549                 )
 550             resolved_candidates = resolved_demands.get(name)
 551             for candidate in resolved_candidates:
 552                 inventory_candidates.append(candidate)
 553             if len(inventory_candidates) < 1:
 554                 if not required_candidates:
 555                     self.triageTranslator.thefinalCallTrans(triage_data_trans)
 556                     raise TranslatorException(
 557                         "Unable to find any candidate for "
 558                         "demand {}".format(name)
 559                     )
 560                 else:
 561                     self.triageTranslator.thefinalCallTrans(triage_data_trans)
 562                     raise TranslatorException(
 563                         "Unable to find any required "
 564                         "candidate for demand {}"
 565                         .format(name)
 566                     )
 567             parsed[name] = {
 568                 "candidates": inventory_candidates,
 569             }
 570         self.triageTranslator.thefinalCallTrans(triage_data_trans)
 571         return parsed
 572
 573     def validate_hpa_constraints(self, req_prop, value):
 574         for para in value.get(req_prop):
 575             # Make sure there is at least one
 576             # set of id, type, directives and flavorProperties
 577             if not para.get('id') \
 578                     or not para.get('type') \
 579                     or not para.get('directives') \
 580                     or not para.get('flavorProperties') \
 581                     or para.get('id') == '' \
 582                     or para.get('type') == '' \
 583                     or not isinstance(para.get('directives'), list) \
 584                     or para.get('flavorProperties') == '':
 585                 raise TranslatorException(
 586                     "HPA requirements need at least "
 587                     "one set of id, type, directives and flavorProperties"
 588                 )
 589             for feature in para.get('flavorProperties'):
 590                 if type(feature) is not dict:
 591                     raise TranslatorException("HPA feature must be a dict")
 592                 # process mandatory parameter
 593                 hpa_mandatory = set(HPA_FEATURES).difference(feature.keys())
 594                 if bool(hpa_mandatory):
 595                     raise TranslatorException(
 596                         "Lack of compulsory elements inside HPA feature")
 597                 # process optional parameter
 598                 hpa_optional = set(feature.keys()).difference(HPA_FEATURES)
 599                 if hpa_optional and not hpa_optional.issubset(HPA_OPTIONAL):
 600                     raise TranslatorException(
 601                         "Got unrecognized elements inside HPA feature")
 602                 if feature.get('mandatory') == 'False' and not feature.get(
 603                         'score'):
 604                     raise TranslatorException(
 605                         "Score needs to be present if mandatory is False")
 606
 607                 for attr in feature.get('hpa-feature-attributes'):
 608                     if type(attr) is not dict:
 609                         raise TranslatorException(
 610                             "HPA feature attributes must be a dict")
 611
 612                     # process mandatory hpa attribute parameter
 613                     hpa_attr_mandatory = set(HPA_ATTRIBUTES).difference(
 614                         attr.keys())
 615                     if bool(hpa_attr_mandatory):
 616                         raise TranslatorException(
 617                             "Lack of compulsory elements inside HPA "
 618                             "feature attributes")
 619                     # process optional hpa attribute parameter
 620                     hpa_attr_optional = set(attr.keys()).difference(
 621                         HPA_ATTRIBUTES)
 622                     if hpa_attr_optional and not hpa_attr_optional.issubset(
 623                             HPA_ATTRIBUTES_OPTIONAL):
 624                         raise TranslatorException(
 625                             "Invalid attributes '{}' found inside HPA "
 626                             "feature attributes".format(hpa_attr_optional))
 627
 628     def parse_constraints(self, constraints):
 629         """Validate/prepare constraints for use by the solver."""
 630         if not isinstance(constraints, dict):
 631             raise TranslatorException("Constraints must be provided in "
 632                                       "dictionary form")
 633
 634         # Look at each constraint. Properties must exist, even if empty.
 635         constraints_copy = copy.deepcopy(constraints)
 636
 637         parsed = {}
 638         for name, constraint in constraints_copy.items():
 639
 640             if not constraint.get('properties'):
 641                 constraint['properties'] = {}
 642
 643             constraint_type = constraint.get('type')
 644             constraint_def = CONSTRAINTS.get(constraint_type)
 645
 646             # Is it a supported type?
 647             if constraint_type not in CONSTRAINTS:
 648                 raise TranslatorException(
 649                     "Unsupported type '{}' found in constraint "
 650                     "named '{}'".format(constraint_type, name))
 651
 652             # Now walk through the constraint's content
 653             for key, value in constraint.items():
 654                 # Must be a supported key
 655                 if key not in CONSTRAINT_KEYS:
 656                     raise TranslatorException(
 657                         "Invalid key '{}' found in constraint "
 658                         "named '{}'".format(key, name))
 659
 660                 # For properties ...
 661                 if key == 'properties':
 662                     # Make sure all required properties are present
 663                     required = constraint_def.get('required', [])
 664                     for req_prop in required:
 665                         if req_prop not in list(value.keys()):
 666                             raise TranslatorException(
 667                                 "Required property '{}' not found in "
 668                                 "constraint named '{}'".format(
 669                                     req_prop, name))
 670                         if not value.get(req_prop) \
 671                                 or value.get(req_prop) == '':
 672                             raise TranslatorException(
 673                                 "No value specified for property '{}' in "
 674                                 "constraint named '{}'".format(
 675                                     req_prop, name))
 676                             # For HPA constraints
 677                         if constraint_type == 'hpa':
 678                             self.validate_hpa_constraints(req_prop, value)
 679
 680                     # Make sure there are no unknown properties
 681                     optional = constraint_def.get('optional', [])
 682                     for prop_name in list(value.keys()):
 683                         if prop_name not in required + optional:
 684                             raise TranslatorException(
 685                                 "Unknown property '{}' in "
 686                                 "constraint named '{}'".format(
 687                                     prop_name, name))
 688
 689                     # If a property has a controlled vocabulary, make
 690                     # sure its value is one of the allowed ones.
 691                     allowed = constraint_def.get('allowed', {})
 692                     for prop_name, allowed_values in allowed.items():
 693                         if prop_name in list(value.keys()):
 694                             prop_value = value.get(prop_name, '')
 695                             if prop_value not in allowed_values:
 696                                 raise TranslatorException(
 697                                     "Property '{}' value '{}' unsupported in "
 698                                     "constraint named '{}' (must be one of "
 699                                     "{})".format(prop_name, prop_value,
 700                                                  name, allowed_values))
 701
 702                     # Break all threshold-formatted values into parts
 703                     thresholds = constraint_def.get('thresholds', {})
 704                     for thr_prop, base_units in thresholds.items():
 705                         if thr_prop in list(value.keys()):
 706                             expression = value.get(thr_prop)
 707                             thr = threshold.Threshold(expression, base_units)
 708                             value[thr_prop] = thr.parts
 709
 710             # We already know we have one or more demands due to
 711             # validate_components(). We still need to coerce the demands
 712             # into a list in case only one demand was provided.
 713             constraint_demands = constraint.get('demands')
 714             if isinstance(constraint_demands, six.string_types):
 715                 constraint['demands'] = [constraint_demands]
 716
 717             # Either split the constraint into parts, one per demand,
 718             # or use it as-is
 719             if constraint_def.get('split'):
 720                 for demand in constraint.get('demands', []):
 721                     constraint_demand = name + '_' + demand
 722                     parsed[constraint_demand] = copy.deepcopy(constraint)
 723                     parsed[constraint_demand]['name'] = name
 724                     parsed[constraint_demand]['demands'] = demand
 725             else:
 726                 parsed[name] = copy.deepcopy(constraint)
 727                 parsed[name]['name'] = name
 728
 729         return parsed
 730
 731     def parse_optimization(self, optimization):
 732         """Validate/prepare optimization for use by the solver."""
 733
 734         # WARNING: The template format for optimization is generalized,
 735         # however the solver is very particular about the expected
 736         # goal, functions, and operands. Therefore, for the time being,
 737         # we are choosing to be highly conservative in what we accept
 738         # at the template level. Once the solver can handle the more
 739         # general form, we can make the translation pass in this
 740         # essentially pre-parsed formula unchanged, or we may allow
 741         # optimizations to be written in algebraic form and pre-parsed
 742         # with antlr4-python2-runtime. (jdandrea 1 Dec 2016)
 743
 744         if not optimization:
 745             LOG.debug("No objective function or "
 746                       "optimzation provided in the template")
 747             return
 748
 749         optimization_copy = copy.deepcopy(optimization)
 750         parsed = {
 751             "goal": "min",
 752             "operation": "sum",
 753             "operands": [],
 754         }
 755
 756         if type(optimization_copy) is not dict:
 757             raise TranslatorException("Optimization must be a dictionary.")
 758
 759         goals = list(optimization_copy.keys())
 760         if goals != ['minimize']:
 761             raise TranslatorException(
 762                 "Optimization must contain a single goal of 'minimize'.")
 763
 764         funcs = list(optimization_copy['minimize'].keys())
 765         if funcs != ['sum']:
 766             raise TranslatorException(
 767                 "Optimization goal 'minimize' must "
 768                 "contain a single function of 'sum'.")
 769         operands = optimization_copy['minimize']['sum']
 770         if type(operands) is not list:
 771             # or len(operands) != 2:
 772             raise TranslatorException(
 773                 "Optimization goal 'minimize', function 'sum' "
 774                 "must be a list of exactly two operands.")
 775
 776         def get_latency_between_args(operand):
 777             args = operand.get('latency_between')
 778             if type(args) is not list and len(args) != 2:
 779                 raise TranslatorException(
 780                     "Optimization 'latency_between' arguments must "
 781                     "be a list of length two.")
 782
 783             got_demand = False
 784             got_location = False
 785             for arg in args:
 786                 if not got_demand and arg in list(self._demands.keys()):
 787                     got_demand = True
 788                 if not got_location and arg in list(self._locations.keys()):
 789                     got_location = True
 790             if not got_demand or not got_location:
 791                 raise TranslatorException(
 792                     "Optimization 'latency_between' arguments {} must "
 793                     "include one valid demand name and one valid "
 794                     "location name.".format(args))
 795
 796             return args
 797
 798         def get_distance_between_args(operand):
 799             args = operand.get('distance_between')
 800             if type(args) is not list and len(args) != 2:
 801                 raise TranslatorException(
 802                     "Optimization 'distance_between' arguments must "
 803                     "be a list of length two.")
 804
 805             got_demand = False
 806             got_location = False
 807             for arg in args:
 808                 if not got_demand and arg in list(self._demands.keys()):
 809                     got_demand = True
 810                 if not got_location and arg in list(self._locations.keys()):
 811                     got_location = True
 812             if not got_demand or not got_location:
 813                 raise TranslatorException(
 814                     "Optimization 'distance_between' arguments {} must "
 815                     "include one valid demand name and one valid "
 816                     "location name.".format(args))
 817
 818             return args
 819
 820         for operand in operands:
 821             weight = 1.0
 822             args = None
 823             nested = False
 824
 825             if list(operand.keys()) == ['distance_between']:
 826                 # Value must be a list of length 2 with one
 827                 # location and one demand
 828                 function = 'distance_between'
 829                 args = get_distance_between_args(operand)
 830
 831             elif list(operand.keys()) == ['product']:
 832                 for product_op in operand['product']:
 833                     if threshold.is_number(product_op):
 834                         weight = product_op
 835                     elif isinstance(product_op, dict):
 836                         if list(product_op.keys()) == ['latency_between']:
 837                             function = 'latency_between'
 838                             args = get_latency_between_args(product_op)
 839                         elif list(product_op.keys()) == ['distance_between']:
 840                             function = 'distance_between'
 841                             args = get_distance_between_args(product_op)
 842                         elif list(product_op.keys()) == ['aic_version']:
 843                             function = 'aic_version'
 844                             args = product_op.get('aic_version')
 845                         elif list(product_op.keys()) == ['hpa_score']:
 846                             function = 'hpa_score'
 847                             args = product_op.get('hpa_score')
 848                             if not self.is_hpa_policy_exists(args):
 849                                 raise TranslatorException(
 850                                     "HPA Score Optimization must include a "
 851                                     "HPA Policy constraint ")
 852                         elif list(product_op.keys()) == ['sum']:
 853                             nested = True
 854                             nested_operands = product_op.get('sum')
 855                             for nested_operand in nested_operands:
 856                                 if list(nested_operand.keys()) == ['product']:
 857                                     nested_weight = weight
 858                                     for nested_product_op in nested_operand['product']:
 859                                         if threshold.is_number(nested_product_op):
 860                                             nested_weight = nested_weight * int(nested_product_op)
 861                                         elif isinstance(nested_product_op, dict):
 862                                             if list(nested_product_op.keys()) == ['latency_between']:
 863                                                 function = 'latency_between'
 864                                                 args = get_latency_between_args(nested_product_op)
 865                                             elif list(nested_product_op.keys()) == ['distance_between']:
 866                                                 function = 'distance_between'
 867                                                 args = get_distance_between_args(nested_product_op)
 868                                     parsed['operands'].append(
 869                                         {
 870                                             "operation": "product",
 871                                             "weight": nested_weight,
 872                                             "function": function,
 873                                             "function_param": args,
 874                                         }
 875                                     )
 876
 877                 if not args:
 878                     raise TranslatorException(
 879                         "Optimization products must include at least "
 880                         "one 'distance_between' function call and "
 881                         "one optional number to be used as a weight.")
 882
 883             # We now have our weight/function_param.
 884             if not nested:
 885                 parsed['operands'].append(
 886                     {
 887                         "operation": "product",
 888                         "weight": weight,
 889                         "function": function,
 890                         "function_param": args,
 891                     }
 892                 )
 893         return parsed
 894
 895     def is_hpa_policy_exists(self, demand_list):
 896         # Check if a HPA constraint exist for the demands in the demand list.
 897         constraints_copy = copy.deepcopy(self._constraints)
 898         for demand in demand_list:
 899             for name, constraint in constraints_copy.items():
 900                 constraint_type = constraint.get('type')
 901                 if constraint_type == 'hpa':
 902                     hpa_demands = constraint.get('demands')
 903                     if demand in hpa_demands:
 904                         return True
 905         return False
 906
 907     def parse_reservations(self, reservations):
 908         demands = self._demands
 909         if not isinstance(reservations, dict):
 910             raise TranslatorException("Reservations must be provided in "
 911                                       "dictionary form")
 912         parsed = {}
 913         if reservations:
 914             parsed['counter'] = 0
 915             parsed['demands'] = {}
 916
 917         for key, value in reservations.items():
 918
 919             if key == "service_model":
 920                 parsed['service_model'] = value
 921
 922             elif key == "service_candidates":
 923                 for name, reservation_details in value.items():
 924                     if not reservation_details.get('properties'):
 925                         reservation_details['properties'] = {}
 926                     for demand in reservation_details.get('demands', []):
 927                         if demand in list(demands.keys()):
 928                             reservation_demand = name + '_' + demand
 929                             parsed['demands'][reservation_demand] = copy.deepcopy(reservation_details)
 930                             parsed['demands'][reservation_demand]['name'] = name
 931                             parsed['demands'][reservation_demand]['demands'] = demand
 932                         else:
 933                             raise TranslatorException("Demand {} must be provided in demands section".format(demand))
 934
 935         return parsed
 936
 937     def do_translation(self):
 938         """Perform the translation."""
 939         if not self.valid:
 940             raise TranslatorException("Can't translate an invalid template.")
 941
 942         request_type = self._parameters.get("request_type") \
 943                        or self._parameters.get("REQUEST_TYPE") \
 944                        or ""
 945
 946         self._translation = {
 947             "conductor_solver": {
 948                 "version": self._version,
 949                 "plan_id": self._plan_id,
 950                 "request_type": request_type,
 951                 "locations": self.parse_locations(self._locations),
 952                 "demands": self.parse_demands(self._demands),
 953                 "objective": self.parse_optimization(self._optmization),
 954                 "constraints": self.parse_constraints(self._constraints),
 955                 "objective": self.parse_optimization(self._optmization),
 956                 "reservations": self.parse_reservations(self._reservations),
 957             }
 958         }
 959
 960     def translate(self):
 961         """Translate the template for the solver."""
 962         self._ok = False
 963         try:
 964             self.create_components()
 965             self.validate_components()
 966             self.parse_parameters()
 967             self.do_translation()
 968             self._ok = True
 969         except Exception as exc:
 970             self._error_message = exc.args
 971
 972     @property
 973     def valid(self):
 974         """Returns True if the template has been validated."""
 975         return self._valid
 976
 977     @property
 978     def ok(self):
 979         """Returns True if the translation was successful."""
 980         return self._ok
 981
 982     @property
 983     def translation(self):
 984         """Returns the translation if it was successful."""
 985         return self._translation
 986
 987     @property
 988     def error_message(self):
 989         """Returns the last known error message."""
 990         return self._error_message
 991
 992
 993 def main():
 994     template_name = 'some_template'
 995
 996     path = os.path.abspath(conductor_root)
 997     dir_path = os.path.dirname(path)
 998
 999     # Prepare service-wide components (e.g., config)
1000     conf = service.prepare_service(
1001         [], config_files=[dir_path + '/../etc/conductor/conductor.conf'])
1002     # conf.set_override('mock', True, 'music_api')
1003
1004     t1 = threshold.Threshold("< 500 ms", "time")
1005     t2 = threshold.Threshold("= 120 mi", "distance")
1006     t3 = threshold.Threshold("160", "currency")
1007     t4 = threshold.Threshold("60-80 Gbps", "throughput")
1008     print('t1: {}\nt2: {}\nt3: {}\nt4: {}\n'.format(t1, t2, t3, t4))
1009
1010     template_file = dir_path + '/tests/data/' + template_name + '.yaml'
1011     fd = open(template_file, "r")
1012     template = yaml.load(fd)
1013
1014     trns = Translator(conf, template_name, str(uuid.uuid4()), template)
1015     trns.translate()
1016     if trns.ok:
1017         print(json.dumps(trns.translation, indent=2))
1018     else:
1019         print("TESTING - Translator Error: {}".format(trns.error_message))
1020
1021
1022 if __name__ == '__main__':
1023     main()