conductor/conductor/controller/translator.py

   1 #
   2 # -------------------------------------------------------------------------
   3 #   Copyright (c) 2015-2017 AT&T Intellectual Property
   4 #   Copyright (C) 2020 Wipro Limited.
   5 #
   6 #   Licensed under the Apache License, Version 2.0 (the "License");
   7 #   you may not use this file except in compliance with the License.
   8 #   You may obtain a copy of the License at
   9 #
  10 #       http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #   Unless required by applicable law or agreed to in writing, software
  13 #   distributed under the License is distributed on an "AS IS" BASIS,
  14 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 #   See the License for the specific language governing permissions and
  16 #   limitations under the License.
  17 #
  18 # -------------------------------------------------------------------------
  19 #
  20
  21 import copy
  22 import datetime
  23 import json
  24 import os
  25 import uuid
  26
  27 import six
  28 import yaml
  29
  30 from conductor import __file__ as conductor_root
  31 from conductor.common.music import messaging as music_messaging
  32 from conductor.common import threshold
  33 from conductor.data.plugins.triage_translator.triage_translator import TraigeTranslator
  34 from conductor.data.plugins.triage_translator.triage_translator_data import TraigeTranslatorData
  35 from conductor import messaging
  36 from conductor import service
  37 from oslo_config import cfg
  38 from oslo_log import log
  39
  40 LOG = log.getLogger(__name__)
  41
  42 CONF = cfg.CONF
  43
  44 VERSIONS = ["2016-11-01", "2017-10-10", "2018-02-01"]
  45 LOCATION_KEYS = ['latitude', 'longitude', 'host_name', 'clli_code']
  46 INVENTORY_PROVIDERS = ['aai', 'generator']
  47 INVENTORY_TYPES = ['cloud', 'service', 'transport', 'vfmodule', 'nssi']
  48 DEFAULT_INVENTORY_PROVIDER = INVENTORY_PROVIDERS[0]
  49 CANDIDATE_KEYS = ['candidate_id', 'cost', 'inventory_type', 'location_id',
  50                   'location_type']
  51 DEMAND_KEYS = ['filtering_attributes', 'passthrough_attributes', 'default_attributes', 'candidates', 'complex',
  52                'conflict_identifier', 'customer_id', 'default_cost', 'excluded_candidates',
  53                'existing_placement', 'flavor', 'inventory_provider',
  54                'inventory_type', 'port_key', 'region', 'required_candidates',
  55                'service_id', 'service_resource_id', 'service_subscription',
  56                'service_type', 'subdivision', 'unique', 'vlan_key']
  57 CONSTRAINT_KEYS = ['type', 'demands', 'properties']
  58 CONSTRAINTS = {
  59     # constraint_type: {
  60     #   split: split into individual constraints, one per demand
  61     #   required: list of required property names,
  62     #   optional: list of optional property names,
  63     #   thresholds: dict of property/base-unit pairs for threshold parsing
  64     #   allowed: dict of keys and allowed values (if controlled vocab);
  65     #            only use this for Conductor-controlled values!
  66     # }
  67     'attribute': {
  68         'split': True,
  69         'required': ['evaluate'],
  70     },
  71     'threshold': {
  72         'split': True,
  73         'required': ['evaluate'],
  74     },
  75     'distance_between_demands': {
  76         'required': ['distance'],
  77         'thresholds': {
  78             'distance': 'distance'
  79         },
  80     },
  81     'distance_to_location': {
  82         'split': True,
  83         'required': ['distance', 'location'],
  84         'thresholds': {
  85             'distance': 'distance'
  86         },
  87     },
  88     'instance_fit': {
  89         'split': True,
  90         'required': ['controller'],
  91         'optional': ['request'],
  92     },
  93     'inventory_group': {},
  94     'region_fit': {
  95         'split': True,
  96         'required': ['controller'],
  97         'optional': ['request'],
  98     },
  99     'zone': {
 100         'required': ['qualifier', 'category'],
 101         'optional': ['location'],
 102         'allowed': {'qualifier': ['same', 'different'],
 103                     'category': ['disaster', 'region', 'complex', 'country',
 104                                  'time', 'maintenance']},
 105     },
 106     'vim_fit': {
 107         'split': True,
 108         'required': ['controller'],
 109         'optional': ['request'],
 110     },
 111     'hpa': {
 112         'split': True,
 113         'required': ['evaluate'],
 114     },
 115 }
 116 HPA_FEATURES = ['architecture', 'hpa-feature', 'hpa-feature-attributes',
 117                 'hpa-version', 'mandatory', 'directives']
 118 HPA_OPTIONAL = ['score']
 119 HPA_ATTRIBUTES = ['hpa-attribute-key', 'hpa-attribute-value', 'operator']
 120 HPA_ATTRIBUTES_OPTIONAL = ['unit']
 121
 122
 123 class TranslatorException(Exception):
 124     pass
 125
 126
 127 class Translator(object):
 128     """Template translator.
 129
 130     Takes an input template and translates it into
 131     something the solver can use. Calls the data service
 132     as needed, giving it the inventory provider as context.
 133     Presently the only inventory provider is A&AI. Others
 134     may be added in the future.
 135     """
 136
 137     def __init__(self, conf, plan_name, plan_id, template):
 138         self.conf = conf
 139         self._template = copy.deepcopy(template)
 140         self._plan_name = plan_name
 141         self._plan_id = plan_id
 142         self._translation = None
 143         self._valid = False
 144         self._ok = False
 145         self.triageTranslatorData = TraigeTranslatorData()
 146         self.triageTranslator = TraigeTranslator()
 147         # Set up the RPC service(s) we want to talk to.
 148         self.data_service = self.setup_rpc(self.conf, "data")
 149
 150     def setup_rpc(self, conf, topic):
 151         """Set up the RPC Client"""
 152         # TODO(jdandrea): Put this pattern inside music_messaging?
 153         transport = messaging.get_transport(conf=conf)
 154         target = music_messaging.Target(topic=topic)
 155         client = music_messaging.RPCClient(conf=conf,
 156                                            transport=transport,
 157                                            target=target)
 158         return client
 159
 160     def create_components(self):
 161         # TODO(jdandrea): Make deep copies so the template is untouched
 162         self._version = self._template.get("homing_template_version")
 163         self._parameters = self._template.get("parameters", {})
 164         self._locations = self._template.get("locations", {})
 165         self._demands = self._template.get("demands", {})
 166         self._constraints = self._template.get("constraints", {})
 167         self._optmization = self._template.get("optimization", {})
 168         self._reservations = self._template.get("reservation", {})
 169
 170         if isinstance(self._version, datetime.date):
 171             self._version = str(self._version)
 172
 173     def validate_components(self):
 174         """Cursory validation of template components.
 175
 176         More detailed validation happens while parsing each component.
 177         """
 178         self._valid = False
 179
 180         # Check version
 181         if self._version not in VERSIONS:
 182             raise TranslatorException(
 183                 "conductor_template_version must be one "
 184                 "of: {}".format(', '.join(VERSIONS)))
 185
 186         # Check top level structure
 187         components = {
 188             "parameters": {
 189                 "name": "Parameter",
 190                 "content": self._parameters,
 191             },
 192             "locations": {
 193                 "name": "Location",
 194                 "keys": LOCATION_KEYS,
 195                 "content": self._locations,
 196             },
 197             "demands": {
 198                 "name": "Demand",
 199                 "content": self._demands,
 200             },
 201             "constraints": {
 202                 "name": "Constraint",
 203                 "keys": CONSTRAINT_KEYS,
 204                 "content": self._constraints,
 205             },
 206             "optimization": {
 207                 "name": "Optimization",
 208                 "content": self._optmization,
 209             },
 210             "reservations": {
 211                 "name": "Reservation",
 212                 "content": self._reservations,
 213             }
 214         }
 215         for name, component in components.items():
 216             name = component.get('name')
 217             keys = component.get('keys', None)
 218             content = component.get('content')
 219
 220             if type(content) is not dict:
 221                 raise TranslatorException(
 222                     "{} section must be a dictionary".format(name))
 223             for content_name, content_def in content.items():
 224                 if not keys:
 225                     continue
 226
 227                 for key in content_def:
 228                     if key not in keys:
 229                         raise TranslatorException(
 230                             "{} {} has an invalid key {}".format(
 231                                 name, content_name, key))
 232
 233         demand_keys = list(self._demands.keys())     # Python 3 Conversion -- dict object to list object
 234         location_keys = list(self._locations.keys())     # Python 3 Conversion -- dict object to list object
 235         for constraint_name, constraint in self._constraints.items():
 236
 237             # Require a single demand (string), or a list of one or more.
 238             demands = constraint.get('demands')
 239             if isinstance(demands, six.string_types):
 240                 demands = [demands]
 241             if not isinstance(demands, list) or len(demands) < 1:
 242                 raise TranslatorException(
 243                     "Demand list for Constraint {} must be "
 244                     "a list of names or a string with one name".format(
 245                         constraint_name))
 246             if not set(demands).issubset(demand_keys + location_keys):
 247                 raise TranslatorException(
 248                     "Undefined Demand(s) {} in Constraint '{}'".format(
 249                         list(set(demands).difference(demand_keys)),
 250                         constraint_name))
 251
 252             properties = constraint.get('properties', None)
 253             if properties:
 254                 location = properties.get('location', None)
 255                 if location:
 256                     if location not in location_keys:
 257                         raise TranslatorException(
 258                             "Location {} in Constraint {} is undefined".format(
 259                                 location, constraint_name))
 260
 261         self._valid = True
 262
 263     def _parse_parameters(self, obj, path=[]):
 264         """Recursively parse all {get_param: X} occurrences
 265
 266         This modifies obj in-place. If you want to keep the original,
 267         pass in a deep copy.
 268         """
 269         # Ok to start with a string ...
 270         if isinstance(path, six.string_types):
 271             # ... but the breadcrumb trail goes in an array.
 272             path = [path]
 273
 274         # Traverse a list
 275         if type(obj) is list:
 276             for idx, val in enumerate(obj, start=0):
 277                 # Add path to the breadcrumb trail
 278                 new_path = list(path)
 279                 new_path[-1] += "[{}]".format(idx)
 280
 281                 # Look at each element.
 282                 obj[idx] = self._parse_parameters(val, new_path)
 283
 284         # Traverse a dict
 285         elif type(obj) is dict:
 286             # Did we find a "{get_param: ...}" intrinsic?
 287             if list(obj.keys()) == ['get_param']:
 288                 param_name = obj['get_param']
 289
 290                 # The parameter name must be a string.
 291                 if not isinstance(param_name, six.string_types):
 292                     path_str = ' > '.join(path)
 293                     raise TranslatorException(
 294                         "Parameter name '{}' not a string in path {}".format(
 295                             param_name, path_str))
 296
 297                 # Parameter name must be defined.
 298                 if param_name not in self._parameters:
 299                     path_str = ' > '.join(path)
 300                     raise TranslatorException(
 301                         "Parameter '{}' undefined in path {}".format(
 302                             param_name, path_str))
 303
 304                 # Return the value in place of the call.
 305                 return self._parameters.get(param_name)
 306
 307             # Not an intrinsic. Traverse as usual.
 308             for key in list(obj.keys()):
 309                 # Add path to the breadcrumb trail.
 310                 new_path = list(path)
 311                 new_path.append(key)
 312
 313                 # Look at each key/value pair.
 314                 obj[key] = self._parse_parameters(obj[key], new_path)
 315
 316         # Return whatever we have after unwinding.
 317         return obj
 318
 319     def parse_parameters(self):
 320         """Resolve all parameters references."""
 321         locations = copy.deepcopy(self._locations)
 322         self._locations = self._parse_parameters(locations, 'locations')
 323
 324         demands = copy.deepcopy(self._demands)
 325         self._demands = self._parse_parameters(demands, 'demands')
 326
 327         constraints = copy.deepcopy(self._constraints)
 328         self._constraints = self._parse_parameters(constraints, 'constraints')
 329
 330         reservations = copy.deepcopy(self._reservations)
 331         self._reservations = self._parse_parameters(reservations,
 332                                                     'reservations')
 333
 334     def parse_locations(self, locations):
 335         """Prepare the locations for use by the solver."""
 336         parsed = {}
 337         for location, args in locations.items():
 338             ctxt = {
 339                 'plan_id': self._plan_id,
 340                 'keyspace': self.conf.keyspace
 341             }
 342
 343             latitude = args.get("latitude")
 344             longitude = args.get("longitude")
 345
 346             if latitude and longitude:
 347                 resolved_location = {"latitude": latitude, "longitude": longitude}
 348             else:
 349                 # ctxt = {}
 350                 response = self.data_service.call(
 351                     ctxt=ctxt,
 352                     method="resolve_location",
 353                     args=args)
 354
 355                 resolved_location = \
 356                     response and response.get('resolved_location')
 357             if not resolved_location:
 358                 raise TranslatorException(
 359                     "Unable to resolve location {}".format(location)
 360                 )
 361             parsed[location] = resolved_location
 362         return parsed
 363
 364     def parse_demands(self, demands):
 365         """Validate/prepare demands for use by the solver."""
 366         if type(demands) is not dict:
 367             raise TranslatorException("Demands must be provided in "
 368                                       "dictionary form")
 369
 370         # Look at each demand
 371         demands_copy = copy.deepcopy(demands)
 372         parsed = {}
 373         for name, requirements in demands_copy.items():
 374             inventory_candidates = []
 375             for requirement in requirements:
 376                 for key in requirement:
 377                     if key not in DEMAND_KEYS:
 378                         raise TranslatorException(
 379                             "Demand {} has an invalid key {}".format(
 380                                 requirement, key))
 381
 382                 if 'candidates' in requirement:
 383                     # Candidates *must* specify an inventory provider
 384                     provider = requirement.get("inventory_provider")
 385                     if provider and provider not in INVENTORY_PROVIDERS:
 386                         raise TranslatorException(
 387                             "Unsupported inventory provider {} "
 388                             "in demand {}".format(provider, name))
 389                     else:
 390                         provider = DEFAULT_INVENTORY_PROVIDER
 391
 392                     # Check each candidate
 393                     for candidate in requirement.get('candidates'):
 394                         # Must be a dictionary
 395                         if type(candidate) is not dict:
 396                             raise TranslatorException(
 397                                 "Candidate found in demand {} that is "
 398                                 "not a dictionary".format(name))
 399
 400                         # Must have only supported keys
 401                         for key in list(candidate.keys()):
 402                             if key not in CANDIDATE_KEYS:
 403                                 raise TranslatorException(
 404                                     "Candidate with invalid key {} found "
 405                                     "in demand {}".format(key, name)
 406                                 )
 407
 408                         # TODO(jdandrea): Check required/optional keys
 409
 410                         # Set the inventory provider if not already
 411                         candidate['inventory_provider'] = \
 412                             candidate.get('inventory_provider', provider)
 413
 414                         # Set cost if not already (default cost is 0?)
 415                         candidate['cost'] = candidate.get('cost', 0)
 416
 417                         # Add to our list of parsed candidates
 418                         inventory_candidates.append(candidate)
 419
 420                 # candidates are specified through inventory providers
 421                 # Do the basic sanity checks for inputs
 422                 else:
 423                     # inventory provider MUST be specified
 424                     provider = requirement.get("inventory_provider")
 425                     if not provider:
 426                         raise TranslatorException(
 427                             "Inventory provider not specified "
 428                             "in demand {}".format(name)
 429                         )
 430                     elif provider and provider not in INVENTORY_PROVIDERS:
 431                         raise TranslatorException(
 432                             "Unsupported inventory provider {} "
 433                             "in demand {}".format(provider, name)
 434                         )
 435                     else:
 436                         provider = DEFAULT_INVENTORY_PROVIDER
 437                         requirement['provider'] = provider
 438
 439                     # inventory type MUST be specified
 440                     inventory_type = requirement.get('inventory_type')
 441                     if not inventory_type or inventory_type == '':
 442                         raise TranslatorException(
 443                             "Inventory type not specified for "
 444                             "demand {}".format(name)
 445                         )
 446                     if inventory_type and \
 447                             inventory_type not in INVENTORY_TYPES:
 448                         raise TranslatorException(
 449                             "Unknown inventory type {} specified for "
 450                             "demand {}".format(inventory_type, name)
 451                         )
 452
 453                     # For service and vfmodule inventories, customer_id and
 454                     # service_type MUST be specified
 455                     if inventory_type == 'service' or inventory_type == 'vfmodule':
 456                         filtering_attributes = requirement.get('filtering_attributes')
 457
 458                         if filtering_attributes:
 459                             customer_id = filtering_attributes.get('customer-id')
 460                             global_customer_id = filtering_attributes.get('global-customer-id')
 461                             if global_customer_id:
 462                                 customer_id = global_customer_id
 463                         else:
 464                             # for backward compatibility
 465                             customer_id = requirement.get('customer_id')
 466                             service_type = requirement.get('service_type')
 467
 468                         if not customer_id:
 469                             raise TranslatorException(
 470                                 "Customer ID not specified for "
 471                                 "demand {}".format(name)
 472                             )
 473                         if not filtering_attributes and not service_type:
 474                             raise TranslatorException(
 475                                 "Service Type not specified for "
 476                                 "demand {}".format(name)
 477                             )
 478
 479                 # TODO(jdandrea): Check required/optional keys for requirement
 480                 # elif 'inventory_type' in requirement:
 481                 #     # For now this is just a stand-in candidate
 482                 #     candidate = {
 483                 #         'inventory_provider':
 484                 #             requirement.get('inventory_provider',
 485                 #                             DEFAULT_INVENTORY_PROVIDER),
 486                 #         'inventory_type':
 487                 #             requirement.get('inventory_type', ''),
 488                 #         'candidate_id': '',
 489                 #         'location_id': '',
 490                 #         'location_type': '',
 491                 #         'cost': 0,
 492                 #     }
 493                 #
 494                 #     # Add to our list of parsed candidates
 495                 #     inventory_candidates.append(candidate)
 496
 497             # Ask conductor-data for one or more candidates.
 498             ctxt = {
 499                 "plan_id": self._plan_id,
 500                 "plan_name": self._plan_name,
 501                 "keyspace": self.conf.keyspace,
 502             }
 503             args = {
 504                 "demands": {
 505                     name: requirements,
 506                 },
 507                 "plan_info": {
 508                     "plan_id": self._plan_id,
 509                     "plan_name": self._plan_name
 510                 },
 511                 "triage_translator_data": self.triageTranslatorData.__dict__
 512
 513             }
 514
 515             # Check if required_candidate and excluded candidate
 516             # are mutually exclusive.
 517             for requirement in requirements:
 518                 required_candidates = requirement.get("required_candidates")
 519                 excluded_candidates = requirement.get("excluded_candidates")
 520                 if (required_candidates and excluded_candidates and set(map(lambda entry: entry['candidate_id'],
 521                                                                         required_candidates))
 522                     & set(map(lambda entry: entry['candidate_id'],
 523                               excluded_candidates))):
 524                     raise TranslatorException(
 525                         "Required candidate list and excluded candidate"
 526                         " list are not mutually exclusive for demand"
 527                         " {}".format(name)
 528                     )
 529             response = self.data_service.call(
 530                 ctxt=ctxt,
 531                 method="resolve_demands",
 532                 args=args)
 533
 534             resolved_demands = \
 535                 response and response.get('resolved_demands')
 536             triage_data_trans = \
 537                 response and response.get('trans')
 538
 539             required_candidates = resolved_demands \
 540                 .get('required_candidates')
 541             if not resolved_demands:
 542                 self.triageTranslator.thefinalCallTrans(triage_data_trans)
 543                 raise TranslatorException(
 544                     "Unable to resolve inventory "
 545                     "candidates for demand {}"
 546                     .format(name)
 547                 )
 548             resolved_candidates = resolved_demands.get(name)
 549             for candidate in resolved_candidates:
 550                 inventory_candidates.append(candidate)
 551             if len(inventory_candidates) < 1:
 552                 if not required_candidates:
 553                     self.triageTranslator.thefinalCallTrans(triage_data_trans)
 554                     raise TranslatorException(
 555                         "Unable to find any candidate for "
 556                         "demand {}".format(name)
 557                     )
 558                 else:
 559                     self.triageTranslator.thefinalCallTrans(triage_data_trans)
 560                     raise TranslatorException(
 561                         "Unable to find any required "
 562                         "candidate for demand {}"
 563                         .format(name)
 564                     )
 565             parsed[name] = {
 566                 "candidates": inventory_candidates,
 567             }
 568         self.triageTranslator.thefinalCallTrans(triage_data_trans)
 569         return parsed
 570
 571     def validate_hpa_constraints(self, req_prop, value):
 572         for para in value.get(req_prop):
 573             # Make sure there is at least one
 574             # set of id, type, directives and flavorProperties
 575             if not para.get('id') \
 576                     or not para.get('type') \
 577                     or not para.get('directives') \
 578                     or not para.get('flavorProperties') \
 579                     or para.get('id') == '' \
 580                     or para.get('type') == '' \
 581                     or not isinstance(para.get('directives'), list) \
 582                     or para.get('flavorProperties') == '':
 583                 raise TranslatorException(
 584                     "HPA requirements need at least "
 585                     "one set of id, type, directives and flavorProperties"
 586                 )
 587             for feature in para.get('flavorProperties'):
 588                 if type(feature) is not dict:
 589                     raise TranslatorException("HPA feature must be a dict")
 590                 # process mandatory parameter
 591                 hpa_mandatory = set(HPA_FEATURES).difference(feature.keys())
 592                 if bool(hpa_mandatory):
 593                     raise TranslatorException(
 594                         "Lack of compulsory elements inside HPA feature")
 595                 # process optional parameter
 596                 hpa_optional = set(feature.keys()).difference(HPA_FEATURES)
 597                 if hpa_optional and not hpa_optional.issubset(HPA_OPTIONAL):
 598                     raise TranslatorException(
 599                         "Got unrecognized elements inside HPA feature")
 600                 if feature.get('mandatory') == 'False' and not feature.get(
 601                         'score'):
 602                     raise TranslatorException(
 603                         "Score needs to be present if mandatory is False")
 604
 605                 for attr in feature.get('hpa-feature-attributes'):
 606                     if type(attr) is not dict:
 607                         raise TranslatorException(
 608                             "HPA feature attributes must be a dict")
 609
 610                     # process mandatory hpa attribute parameter
 611                     hpa_attr_mandatory = set(HPA_ATTRIBUTES).difference(
 612                         attr.keys())
 613                     if bool(hpa_attr_mandatory):
 614                         raise TranslatorException(
 615                             "Lack of compulsory elements inside HPA "
 616                             "feature attributes")
 617                     # process optional hpa attribute parameter
 618                     hpa_attr_optional = set(attr.keys()).difference(
 619                         HPA_ATTRIBUTES)
 620                     if hpa_attr_optional and not hpa_attr_optional.issubset(
 621                             HPA_ATTRIBUTES_OPTIONAL):
 622                         raise TranslatorException(
 623                             "Invalid attributes '{}' found inside HPA "
 624                             "feature attributes".format(hpa_attr_optional))
 625
 626     def parse_constraints(self, constraints):
 627         """Validate/prepare constraints for use by the solver."""
 628         if not isinstance(constraints, dict):
 629             raise TranslatorException("Constraints must be provided in "
 630                                       "dictionary form")
 631
 632         # Look at each constraint. Properties must exist, even if empty.
 633         constraints_copy = copy.deepcopy(constraints)
 634
 635         parsed = {}
 636         for name, constraint in constraints_copy.items():
 637
 638             if not constraint.get('properties'):
 639                 constraint['properties'] = {}
 640
 641             constraint_type = constraint.get('type')
 642             constraint_def = CONSTRAINTS.get(constraint_type)
 643
 644             # Is it a supported type?
 645             if constraint_type not in CONSTRAINTS:
 646                 raise TranslatorException(
 647                     "Unsupported type '{}' found in constraint "
 648                     "named '{}'".format(constraint_type, name))
 649
 650             # Now walk through the constraint's content
 651             for key, value in constraint.items():
 652                 # Must be a supported key
 653                 if key not in CONSTRAINT_KEYS:
 654                     raise TranslatorException(
 655                         "Invalid key '{}' found in constraint "
 656                         "named '{}'".format(key, name))
 657
 658                 # For properties ...
 659                 if key == 'properties':
 660                     # Make sure all required properties are present
 661                     required = constraint_def.get('required', [])
 662                     for req_prop in required:
 663                         if req_prop not in list(value.keys()):
 664                             raise TranslatorException(
 665                                 "Required property '{}' not found in "
 666                                 "constraint named '{}'".format(
 667                                     req_prop, name))
 668                         if not value.get(req_prop) \
 669                                 or value.get(req_prop) == '':
 670                             raise TranslatorException(
 671                                 "No value specified for property '{}' in "
 672                                 "constraint named '{}'".format(
 673                                     req_prop, name))
 674                             # For HPA constraints
 675                         if constraint_type == 'hpa':
 676                             self.validate_hpa_constraints(req_prop, value)
 677
 678                     # Make sure there are no unknown properties
 679                     optional = constraint_def.get('optional', [])
 680                     for prop_name in list(value.keys()):
 681                         if prop_name not in required + optional:
 682                             raise TranslatorException(
 683                                 "Unknown property '{}' in "
 684                                 "constraint named '{}'".format(
 685                                     prop_name, name))
 686
 687                     # If a property has a controlled vocabulary, make
 688                     # sure its value is one of the allowed ones.
 689                     allowed = constraint_def.get('allowed', {})
 690                     for prop_name, allowed_values in allowed.items():
 691                         if prop_name in list(value.keys()):
 692                             prop_value = value.get(prop_name, '')
 693                             if prop_value not in allowed_values:
 694                                 raise TranslatorException(
 695                                     "Property '{}' value '{}' unsupported in "
 696                                     "constraint named '{}' (must be one of "
 697                                     "{})".format(prop_name, prop_value,
 698                                                  name, allowed_values))
 699
 700                     # Break all threshold-formatted values into parts
 701                     thresholds = constraint_def.get('thresholds', {})
 702                     for thr_prop, base_units in thresholds.items():
 703                         if thr_prop in list(value.keys()):
 704                             expression = value.get(thr_prop)
 705                             thr = threshold.Threshold(expression, base_units)
 706                             value[thr_prop] = thr.parts
 707
 708             # We already know we have one or more demands due to
 709             # validate_components(). We still need to coerce the demands
 710             # into a list in case only one demand was provided.
 711             constraint_demands = constraint.get('demands')
 712             if isinstance(constraint_demands, six.string_types):
 713                 constraint['demands'] = [constraint_demands]
 714
 715             # Either split the constraint into parts, one per demand,
 716             # or use it as-is
 717             if constraint_def.get('split'):
 718                 for demand in constraint.get('demands', []):
 719                     constraint_demand = name + '_' + demand
 720                     parsed[constraint_demand] = copy.deepcopy(constraint)
 721                     parsed[constraint_demand]['name'] = name
 722                     parsed[constraint_demand]['demands'] = demand
 723             else:
 724                 parsed[name] = copy.deepcopy(constraint)
 725                 parsed[name]['name'] = name
 726
 727         return parsed
 728
 729     def parse_optimization(self, optimization):
 730         """Validate/prepare optimization for use by the solver."""
 731
 732         # WARNING: The template format for optimization is generalized,
 733         # however the solver is very particular about the expected
 734         # goal, functions, and operands. Therefore, for the time being,
 735         # we are choosing to be highly conservative in what we accept
 736         # at the template level. Once the solver can handle the more
 737         # general form, we can make the translation pass in this
 738         # essentially pre-parsed formula unchanged, or we may allow
 739         # optimizations to be written in algebraic form and pre-parsed
 740         # with antlr4-python2-runtime. (jdandrea 1 Dec 2016)
 741
 742         if not optimization:
 743             LOG.debug("No objective function or "
 744                       "optimzation provided in the template")
 745             return
 746
 747         optimization_copy = copy.deepcopy(optimization)
 748         parsed = {
 749             "goal": "min",
 750             "operation": "sum",
 751             "operands": [],
 752         }
 753
 754         if type(optimization_copy) is not dict:
 755             raise TranslatorException("Optimization must be a dictionary.")
 756
 757         goals = list(optimization_copy.keys())
 758         if goals != ['minimize']:
 759             raise TranslatorException(
 760                 "Optimization must contain a single goal of 'minimize'.")
 761
 762         funcs = list(optimization_copy['minimize'].keys())
 763         if funcs != ['sum']:
 764             raise TranslatorException(
 765                 "Optimization goal 'minimize' must "
 766                 "contain a single function of 'sum'.")
 767         operands = optimization_copy['minimize']['sum']
 768         if type(operands) is not list:
 769             # or len(operands) != 2:
 770             raise TranslatorException(
 771                 "Optimization goal 'minimize', function 'sum' "
 772                 "must be a list of exactly two operands.")
 773
 774         def get_latency_between_args(operand):
 775             args = operand.get('latency_between')
 776             if type(args) is not list and len(args) != 2:
 777                 raise TranslatorException(
 778                     "Optimization 'latency_between' arguments must "
 779                     "be a list of length two.")
 780
 781             got_demand = False
 782             got_location = False
 783             for arg in args:
 784                 if not got_demand and arg in list(self._demands.keys()):
 785                     got_demand = True
 786                 if not got_location and arg in list(self._locations.keys()):
 787                     got_location = True
 788             if not got_demand or not got_location:
 789                 raise TranslatorException(
 790                     "Optimization 'latency_between' arguments {} must "
 791                     "include one valid demand name and one valid "
 792                     "location name.".format(args))
 793
 794             return args
 795
 796         def get_distance_between_args(operand):
 797             args = operand.get('distance_between')
 798             if type(args) is not list and len(args) != 2:
 799                 raise TranslatorException(
 800                     "Optimization 'distance_between' arguments must "
 801                     "be a list of length two.")
 802
 803             got_demand = False
 804             got_location = False
 805             for arg in args:
 806                 if not got_demand and arg in list(self._demands.keys()):
 807                     got_demand = True
 808                 if not got_location and arg in list(self._locations.keys()):
 809                     got_location = True
 810             if not got_demand or not got_location:
 811                 raise TranslatorException(
 812                     "Optimization 'distance_between' arguments {} must "
 813                     "include one valid demand name and one valid "
 814                     "location name.".format(args))
 815
 816             return args
 817
 818         for operand in operands:
 819             weight = 1.0
 820             args = None
 821             nested = False
 822
 823             if list(operand.keys()) == ['distance_between']:
 824                 # Value must be a list of length 2 with one
 825                 # location and one demand
 826                 function = 'distance_between'
 827                 args = get_distance_between_args(operand)
 828
 829             elif list(operand.keys()) == ['product']:
 830                 for product_op in operand['product']:
 831                     if threshold.is_number(product_op):
 832                         weight = product_op
 833                     elif isinstance(product_op, dict):
 834                         if list(product_op.keys()) == ['latency_between']:
 835                             function = 'latency_between'
 836                             args = get_latency_between_args(product_op)
 837                         elif list(product_op.keys()) == ['distance_between']:
 838                             function = 'distance_between'
 839                             args = get_distance_between_args(product_op)
 840                         elif list(product_op.keys()) == ['aic_version']:
 841                             function = 'aic_version'
 842                             args = product_op.get('aic_version')
 843                         elif list(product_op.keys()) == ['hpa_score']:
 844                             function = 'hpa_score'
 845                             args = product_op.get('hpa_score')
 846                             if not self.is_hpa_policy_exists(args):
 847                                 raise TranslatorException(
 848                                     "HPA Score Optimization must include a "
 849                                     "HPA Policy constraint ")
 850                         elif list(product_op.keys()) == ['sum']:
 851                             nested = True
 852                             nested_operands = product_op.get('sum')
 853                             for nested_operand in nested_operands:
 854                                 if list(nested_operand.keys()) == ['product']:
 855                                     nested_weight = weight
 856                                     for nested_product_op in nested_operand['product']:
 857                                         if threshold.is_number(nested_product_op):
 858                                             nested_weight = nested_weight * int(nested_product_op)
 859                                         elif isinstance(nested_product_op, dict):
 860                                             if list(nested_product_op.keys()) == ['latency_between']:
 861                                                 function = 'latency_between'
 862                                                 args = get_latency_between_args(nested_product_op)
 863                                             elif list(nested_product_op.keys()) == ['distance_between']:
 864                                                 function = 'distance_between'
 865                                                 args = get_distance_between_args(nested_product_op)
 866                                     parsed['operands'].append(
 867                                         {
 868                                             "operation": "product",
 869                                             "weight": nested_weight,
 870                                             "function": function,
 871                                             "function_param": args,
 872                                         }
 873                                     )
 874
 875                 if not args:
 876                     raise TranslatorException(
 877                         "Optimization products must include at least "
 878                         "one 'distance_between' function call and "
 879                         "one optional number to be used as a weight.")
 880
 881             # We now have our weight/function_param.
 882             if not nested:
 883                 parsed['operands'].append(
 884                     {
 885                         "operation": "product",
 886                         "weight": weight,
 887                         "function": function,
 888                         "function_param": args,
 889                     }
 890                 )
 891         return parsed
 892
 893     def is_hpa_policy_exists(self, demand_list):
 894         # Check if a HPA constraint exist for the demands in the demand list.
 895         constraints_copy = copy.deepcopy(self._constraints)
 896         for demand in demand_list:
 897             for name, constraint in constraints_copy.items():
 898                 constraint_type = constraint.get('type')
 899                 if constraint_type == 'hpa':
 900                     hpa_demands = constraint.get('demands')
 901                     if demand in hpa_demands:
 902                         return True
 903         return False
 904
 905     def parse_reservations(self, reservations):
 906         demands = self._demands
 907         if not isinstance(reservations, dict):
 908             raise TranslatorException("Reservations must be provided in "
 909                                       "dictionary form")
 910         parsed = {}
 911         if reservations:
 912             parsed['counter'] = 0
 913             parsed['demands'] = {}
 914
 915         for key, value in reservations.items():
 916
 917             if key == "service_model":
 918                 parsed['service_model'] = value
 919
 920             elif key == "service_candidates":
 921                 for name, reservation_details in value.items():
 922                     if not reservation_details.get('properties'):
 923                         reservation_details['properties'] = {}
 924                     for demand in reservation_details.get('demands', []):
 925                         if demand in list(demands.keys()):
 926                             reservation_demand = name + '_' + demand
 927                             parsed['demands'][reservation_demand] = copy.deepcopy(reservation_details)
 928                             parsed['demands'][reservation_demand]['name'] = name
 929                             parsed['demands'][reservation_demand]['demands'] = demand
 930                         else:
 931                             raise TranslatorException("Demand {} must be provided in demands section".format(demand))
 932
 933         return parsed
 934
 935     def do_translation(self):
 936         """Perform the translation."""
 937         if not self.valid:
 938             raise TranslatorException("Can't translate an invalid template.")
 939
 940         request_type = self._parameters.get("request_type") or self._parameters.get("REQUEST_TYPE") or ""
 941
 942         self._translation = {
 943             "conductor_solver": {
 944                 "version": self._version,
 945                 "plan_id": self._plan_id,
 946                 "request_type": request_type,
 947                 "locations": self.parse_locations(self._locations),
 948                 "demands": self.parse_demands(self._demands),
 949                 "objective": self.parse_optimization(self._optmization),
 950                 "constraints": self.parse_constraints(self._constraints),
 951                 "reservations": self.parse_reservations(self._reservations),
 952             }
 953         }
 954
 955     def translate(self):
 956         """Translate the template for the solver."""
 957         self._ok = False
 958         try:
 959             self.create_components()
 960             self.validate_components()
 961             self.parse_parameters()
 962             self.do_translation()
 963             self._ok = True
 964         except Exception as exc:
 965             self._error_message = exc.args
 966
 967     @property
 968     def valid(self):
 969         """Returns True if the template has been validated."""
 970         return self._valid
 971
 972     @property
 973     def ok(self):
 974         """Returns True if the translation was successful."""
 975         return self._ok
 976
 977     @property
 978     def translation(self):
 979         """Returns the translation if it was successful."""
 980         return self._translation
 981
 982     @property
 983     def error_message(self):
 984         """Returns the last known error message."""
 985         return self._error_message
 986
 987
 988 def main():
 989     template_name = 'some_template'
 990
 991     path = os.path.abspath(conductor_root)
 992     dir_path = os.path.dirname(path)
 993
 994     # Prepare service-wide components (e.g., config)
 995     conf = service.prepare_service(
 996         [], config_files=[dir_path + '/../etc/conductor/conductor.conf'])
 997     # conf.set_override('mock', True, 'music_api')
 998
 999     t1 = threshold.Threshold("< 500 ms", "time")
1000     t2 = threshold.Threshold("= 120 mi", "distance")
1001     t3 = threshold.Threshold("160", "currency")
1002     t4 = threshold.Threshold("60-80 Gbps", "throughput")
1003     print('t1: {}\nt2: {}\nt3: {}\nt4: {}\n'.format(t1, t2, t3, t4))
1004
1005     template_file = dir_path + '/tests/data/' + template_name + '.yaml'
1006     fd = open(template_file, "r")
1007     template = yaml.load(fd)
1008
1009     trns = Translator(conf, template_name, str(uuid.uuid4()), template)
1010     trns.translate()
1011     if trns.ok:
1012         print(json.dumps(trns.translation, indent=2))
1013     else:
1014         print("TESTING - Translator Error: {}".format(trns.error_message))
1015
1016
1017 if __name__ == '__main__':
1018     main()