conductor/conductor/controller/translator.py

   1 #
   2 # -------------------------------------------------------------------------
   3 #   Copyright (c) 2015-2017 AT&T Intellectual Property
   4 #   Copyright (C) 2020 Wipro Limited.
   5 #
   6 #   Licensed under the Apache License, Version 2.0 (the "License");
   7 #   you may not use this file except in compliance with the License.
   8 #   You may obtain a copy of the License at
   9 #
  10 #       http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #   Unless required by applicable law or agreed to in writing, software
  13 #   distributed under the License is distributed on an "AS IS" BASIS,
  14 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 #   See the License for the specific language governing permissions and
  16 #   limitations under the License.
  17 #
  18 # -------------------------------------------------------------------------
  19 #
  20
  21 import copy
  22 import datetime
  23 import json
  24 import os
  25 import uuid
  26
  27 import six
  28 import yaml
  29 from conductor import __file__ as conductor_root
  30 from conductor import messaging
  31 from conductor import service
  32
  33 from conductor.common import threshold
  34 from conductor.common.music import messaging as music_messaging
  35 from conductor.data.plugins.triage_translator.triage_translator_data import TraigeTranslatorData
  36 from conductor.data.plugins.triage_translator.triage_translator import TraigeTranslator
  37 from oslo_config import cfg
  38 from oslo_log import log
  39
  40 LOG = log.getLogger(__name__)
  41
  42 CONF = cfg.CONF
  43
  44 VERSIONS = ["2016-11-01", "2017-10-10", "2018-02-01"]
  45 LOCATION_KEYS = ['latitude', 'longitude', 'host_name', 'clli_code']
  46 INVENTORY_PROVIDERS = ['aai']
  47 INVENTORY_TYPES = ['cloud', 'service', 'transport', 'vfmodule', 'nssi']
  48 DEFAULT_INVENTORY_PROVIDER = INVENTORY_PROVIDERS[0]
  49 CANDIDATE_KEYS = ['candidate_id', 'cost', 'inventory_type', 'location_id',
  50                   'location_type']
  51 DEMAND_KEYS = ['filtering_attributes', 'passthrough_attributes', 'candidates', 'complex', 'conflict_identifier',
  52                'customer_id', 'default_cost', 'excluded_candidates',
  53                'existing_placement', 'flavor', 'inventory_provider',
  54                'inventory_type', 'port_key', 'region', 'required_candidates',
  55                'service_id', 'service_resource_id', 'service_subscription',
  56                'service_type', 'subdivision', 'unique', 'vlan_key']
  57 CONSTRAINT_KEYS = ['type', 'demands', 'properties']
  58 CONSTRAINTS = {
  59     # constraint_type: {
  60     #   split: split into individual constraints, one per demand
  61     #   required: list of required property names,
  62     #   optional: list of optional property names,
  63     #   thresholds: dict of property/base-unit pairs for threshold parsing
  64     #   allowed: dict of keys and allowed values (if controlled vocab);
  65     #            only use this for Conductor-controlled values!
  66     # }
  67     'attribute': {
  68         'split': True,
  69         'required': ['evaluate'],
  70     },
  71     'threshold': {
  72         'split': True,
  73         'required': ['attribute', 'threshold', 'operator'],
  74         'optional': ['unit']
  75     },
  76     'distance_between_demands': {
  77         'required': ['distance'],
  78         'thresholds': {
  79             'distance': 'distance'
  80         },
  81     },
  82     'distance_to_location': {
  83         'split': True,
  84         'required': ['distance', 'location'],
  85         'thresholds': {
  86             'distance': 'distance'
  87         },
  88     },
  89     'instance_fit': {
  90         'split': True,
  91         'required': ['controller'],
  92         'optional': ['request'],
  93     },
  94     'inventory_group': {},
  95     'region_fit': {
  96         'split': True,
  97         'required': ['controller'],
  98         'optional': ['request'],
  99     },
 100     'zone': {
 101         'required': ['qualifier', 'category'],
 102         'optional': ['location'],
 103         'allowed': {'qualifier': ['same', 'different'],
 104                     'category': ['disaster', 'region', 'complex', 'country',
 105                                  'time', 'maintenance']},
 106     },
 107     'vim_fit': {
 108         'split': True,
 109         'required': ['controller'],
 110         'optional': ['request'],
 111     },
 112     'hpa': {
 113         'split': True,
 114         'required': ['evaluate'],
 115     },
 116 }
 117 HPA_FEATURES = ['architecture', 'hpa-feature', 'hpa-feature-attributes',
 118                 'hpa-version', 'mandatory', 'directives']
 119 HPA_OPTIONAL = ['score']
 120 HPA_ATTRIBUTES = ['hpa-attribute-key', 'hpa-attribute-value', 'operator']
 121 HPA_ATTRIBUTES_OPTIONAL = ['unit']
 122
 123
 124 class TranslatorException(Exception):
 125     pass
 126
 127
 128 class Translator(object):
 129     """Template translator.
 130
 131     Takes an input template and translates it into
 132     something the solver can use. Calls the data service
 133     as needed, giving it the inventory provider as context.
 134     Presently the only inventory provider is A&AI. Others
 135     may be added in the future.
 136     """
 137
 138     def __init__(self, conf, plan_name, plan_id, template):
 139         self.conf = conf
 140         self._template = copy.deepcopy(template)
 141         self._plan_name = plan_name
 142         self._plan_id = plan_id
 143         self._translation = None
 144         self._valid = False
 145         self._ok = False
 146         self.triageTranslatorData= TraigeTranslatorData()
 147         self.triageTranslator = TraigeTranslator()
 148         # Set up the RPC service(s) we want to talk to.
 149         self.data_service = self.setup_rpc(self.conf, "data")
 150
 151     def setup_rpc(self, conf, topic):
 152         """Set up the RPC Client"""
 153         # TODO(jdandrea): Put this pattern inside music_messaging?
 154         transport = messaging.get_transport(conf=conf)
 155         target = music_messaging.Target(topic=topic)
 156         client = music_messaging.RPCClient(conf=conf,
 157                                            transport=transport,
 158                                            target=target)
 159         return client
 160
 161     def create_components(self):
 162         # TODO(jdandrea): Make deep copies so the template is untouched
 163         self._version = self._template.get("homing_template_version")
 164         self._parameters = self._template.get("parameters", {})
 165         self._locations = self._template.get("locations", {})
 166         self._demands = self._template.get("demands", {})
 167         self._constraints = self._template.get("constraints", {})
 168         self._optmization = self._template.get("optimization", {})
 169         self._reservations = self._template.get("reservation", {})
 170
 171         if isinstance(self._version, datetime.date):
 172             self._version = str(self._version)
 173
 174     def validate_components(self):
 175         """Cursory validation of template components.
 176
 177         More detailed validation happens while parsing each component.
 178         """
 179         self._valid = False
 180
 181         # Check version
 182         if self._version not in VERSIONS:
 183             raise TranslatorException(
 184                 "conductor_template_version must be one "
 185                 "of: {}".format(', '.join(VERSIONS)))
 186
 187         # Check top level structure
 188         components = {
 189             "parameters": {
 190                 "name": "Parameter",
 191                 "content": self._parameters,
 192             },
 193             "locations": {
 194                 "name": "Location",
 195                 "keys": LOCATION_KEYS,
 196                 "content": self._locations,
 197             },
 198             "demands": {
 199                 "name": "Demand",
 200                 "content": self._demands,
 201             },
 202             "constraints": {
 203                 "name": "Constraint",
 204                 "keys": CONSTRAINT_KEYS,
 205                 "content": self._constraints,
 206             },
 207             "optimization": {
 208                 "name": "Optimization",
 209                 "content": self._optmization,
 210             },
 211             "reservations": {
 212                 "name": "Reservation",
 213                 "content": self._reservations,
 214             }
 215         }
 216         for name, component in components.items():
 217             name = component.get('name')
 218             keys = component.get('keys', None)
 219             content = component.get('content')
 220
 221             if type(content) is not dict:
 222                 raise TranslatorException(
 223                     "{} section must be a dictionary".format(name))
 224             for content_name, content_def in content.items():
 225                 if not keys:
 226                     continue
 227
 228                 for key in content_def:
 229                     if key not in keys:
 230                         raise TranslatorException(
 231                             "{} {} has an invalid key {}".format(
 232                                 name, content_name, key))
 233
 234         demand_keys = list(self._demands.keys())     # Python 3 Conversion -- dict object to list object
 235         location_keys = list(self._locations.keys())     # Python 3 Conversion -- dict object to list object
 236         for constraint_name, constraint in self._constraints.items():
 237
 238             # Require a single demand (string), or a list of one or more.
 239             demands = constraint.get('demands')
 240             if isinstance(demands, six.string_types):
 241                 demands = [demands]
 242             if not isinstance(demands, list) or len(demands) < 1:
 243                 raise TranslatorException(
 244                     "Demand list for Constraint {} must be "
 245                     "a list of names or a string with one name".format(
 246                         constraint_name))
 247             if not set(demands).issubset(demand_keys + location_keys):
 248                 raise TranslatorException(
 249                     "Undefined Demand(s) {} in Constraint '{}'".format(
 250                         list(set(demands).difference(demand_keys)),
 251                         constraint_name))
 252
 253             properties = constraint.get('properties', None)
 254             if properties:
 255                 location = properties.get('location', None)
 256                 if location:
 257                     if location not in location_keys:
 258                         raise TranslatorException(
 259                             "Location {} in Constraint {} is undefined".format(
 260                                 location, constraint_name))
 261
 262         self._valid = True
 263
 264     def _parse_parameters(self, obj, path=[]):
 265         """Recursively parse all {get_param: X} occurrences
 266
 267         This modifies obj in-place. If you want to keep the original,
 268         pass in a deep copy.
 269         """
 270         # Ok to start with a string ...
 271         if isinstance(path, six.string_types):
 272             # ... but the breadcrumb trail goes in an array.
 273             path = [path]
 274
 275         # Traverse a list
 276         if type(obj) is list:
 277             for idx, val in enumerate(obj, start=0):
 278                 # Add path to the breadcrumb trail
 279                 new_path = list(path)
 280                 new_path[-1] += "[{}]".format(idx)
 281
 282                 # Look at each element.
 283                 obj[idx] = self._parse_parameters(val, new_path)
 284
 285         # Traverse a dict
 286         elif type(obj) is dict:
 287             # Did we find a "{get_param: ...}" intrinsic?
 288             if list(obj.keys()) == ['get_param']:
 289                 param_name = obj['get_param']
 290
 291                 # The parameter name must be a string.
 292                 if not isinstance(param_name, six.string_types):
 293                     path_str = ' > '.join(path)
 294                     raise TranslatorException(
 295                         "Parameter name '{}' not a string in path {}".format(
 296                             param_name, path_str))
 297
 298                 # Parameter name must be defined.
 299                 if param_name not in self._parameters:
 300                     path_str = ' > '.join(path)
 301                     raise TranslatorException(
 302                         "Parameter '{}' undefined in path {}".format(
 303                             param_name, path_str))
 304
 305                 # Return the value in place of the call.
 306                 return self._parameters.get(param_name)
 307
 308             # Not an intrinsic. Traverse as usual.
 309             for key in list(obj.keys()):
 310                 # Add path to the breadcrumb trail.
 311                 new_path = list(path)
 312                 new_path.append(key)
 313
 314                 # Look at each key/value pair.
 315                 obj[key] = self._parse_parameters(obj[key], new_path)
 316
 317         # Return whatever we have after unwinding.
 318         return obj
 319
 320     def parse_parameters(self):
 321         """Resolve all parameters references."""
 322         locations = copy.deepcopy(self._locations)
 323         self._locations = self._parse_parameters(locations, 'locations')
 324
 325         demands = copy.deepcopy(self._demands)
 326         self._demands = self._parse_parameters(demands, 'demands')
 327
 328         constraints = copy.deepcopy(self._constraints)
 329         self._constraints = self._parse_parameters(constraints, 'constraints')
 330
 331         reservations = copy.deepcopy(self._reservations)
 332         self._reservations = self._parse_parameters(reservations,
 333                                                     'reservations')
 334
 335     def parse_locations(self, locations):
 336         """Prepare the locations for use by the solver."""
 337         parsed = {}
 338         for location, args in locations.items():
 339             ctxt = {
 340                 'plan_id': self._plan_id,
 341                 'keyspace': self.conf.keyspace
 342             }
 343
 344             latitude = args.get("latitude")
 345             longitude = args.get("longitude")
 346
 347             if latitude and longitude:
 348                 resolved_location = {"latitude": latitude, "longitude": longitude}
 349             else:
 350                 # ctxt = {}
 351                 response = self.data_service.call(
 352                     ctxt=ctxt,
 353                     method="resolve_location",
 354                     args=args)
 355
 356                 resolved_location = \
 357                     response and response.get('resolved_location')
 358             if not resolved_location:
 359                 raise TranslatorException(
 360                     "Unable to resolve location {}".format(location)
 361                 )
 362             parsed[location] = resolved_location
 363         return parsed
 364
 365     def parse_demands(self, demands):
 366         """Validate/prepare demands for use by the solver."""
 367         if type(demands) is not dict:
 368             raise TranslatorException("Demands must be provided in "
 369                                       "dictionary form")
 370
 371         # Look at each demand
 372         demands_copy = copy.deepcopy(demands)
 373         parsed = {}
 374         for name, requirements in demands_copy.items():
 375             inventory_candidates = []
 376             for requirement in requirements:
 377                 for key in requirement:
 378                     if key not in DEMAND_KEYS:
 379                         raise TranslatorException(
 380                             "Demand {} has an invalid key {}".format(
 381                                 requirement, key))
 382
 383                 if 'candidates' in requirement:
 384                     # Candidates *must* specify an inventory provider
 385                     provider = requirement.get("inventory_provider")
 386                     if provider and provider not in INVENTORY_PROVIDERS:
 387                         raise TranslatorException(
 388                             "Unsupported inventory provider {} "
 389                             "in demand {}".format(provider, name))
 390                     else:
 391                         provider = DEFAULT_INVENTORY_PROVIDER
 392
 393                     # Check each candidate
 394                     for candidate in requirement.get('candidates'):
 395                         # Must be a dictionary
 396                         if type(candidate) is not dict:
 397                             raise TranslatorException(
 398                                 "Candidate found in demand {} that is "
 399                                 "not a dictionary".format(name))
 400
 401                         # Must have only supported keys
 402                         for key in list(candidate.keys()):
 403                             if key not in CANDIDATE_KEYS:
 404                                 raise TranslatorException(
 405                                     "Candidate with invalid key {} found "
 406                                     "in demand {}".format(key, name)
 407                                 )
 408
 409                         # TODO(jdandrea): Check required/optional keys
 410
 411                         # Set the inventory provider if not already
 412                         candidate['inventory_provider'] = \
 413                             candidate.get('inventory_provider', provider)
 414
 415                         # Set cost if not already (default cost is 0?)
 416                         candidate['cost'] = candidate.get('cost', 0)
 417
 418                         # Add to our list of parsed candidates
 419                         inventory_candidates.append(candidate)
 420
 421                 # candidates are specified through inventory providers
 422                 # Do the basic sanity checks for inputs
 423                 else:
 424                     # inventory provider MUST be specified
 425                     provider = requirement.get("inventory_provider")
 426                     if not provider:
 427                         raise TranslatorException(
 428                             "Inventory provider not specified "
 429                             "in demand {}".format(name)
 430                         )
 431                     elif provider and provider not in INVENTORY_PROVIDERS:
 432                         raise TranslatorException(
 433                             "Unsupported inventory provider {} "
 434                             "in demand {}".format(provider, name)
 435                         )
 436                     else:
 437                         provider = DEFAULT_INVENTORY_PROVIDER
 438                         requirement['provider'] = provider
 439
 440                     # inventory type MUST be specified
 441                     inventory_type = requirement.get('inventory_type')
 442                     if not inventory_type or inventory_type == '':
 443                         raise TranslatorException(
 444                             "Inventory type not specified for "
 445                             "demand {}".format(name)
 446                         )
 447                     if inventory_type and \
 448                             inventory_type not in INVENTORY_TYPES:
 449                         raise TranslatorException(
 450                             "Unknown inventory type {} specified for "
 451                             "demand {}".format(inventory_type, name)
 452                         )
 453
 454                     # For service and vfmodule inventories, customer_id and
 455                     # service_type MUST be specified
 456                     if inventory_type == 'service' or inventory_type == 'vfmodule':
 457                         filtering_attributes = requirement.get('filtering_attributes')
 458
 459                         if filtering_attributes:
 460                             customer_id = filtering_attributes.get('customer-id')
 461                             global_customer_id = filtering_attributes.get('global-customer-id')
 462                             if global_customer_id:
 463                                 customer_id = global_customer_id
 464                         else:
 465                             # for backward compatibility
 466                             customer_id = requirement.get('customer_id')
 467                             service_type = requirement.get('service_type')
 468
 469                         if not customer_id:
 470                             raise TranslatorException(
 471                                 "Customer ID not specified for "
 472                                 "demand {}".format(name)
 473                             )
 474                         if not filtering_attributes and not service_type:
 475                             raise TranslatorException(
 476                                 "Service Type not specified for "
 477                                 "demand {}".format(name)
 478                             )
 479
 480                 # TODO(jdandrea): Check required/optional keys for requirement
 481                 # elif 'inventory_type' in requirement:
 482                 #     # For now this is just a stand-in candidate
 483                 #     candidate = {
 484                 #         'inventory_provider':
 485                 #             requirement.get('inventory_provider',
 486                 #                             DEFAULT_INVENTORY_PROVIDER),
 487                 #         'inventory_type':
 488                 #             requirement.get('inventory_type', ''),
 489                 #         'candidate_id': '',
 490                 #         'location_id': '',
 491                 #         'location_type': '',
 492                 #         'cost': 0,
 493                 #     }
 494                 #
 495                 #     # Add to our list of parsed candidates
 496                 #     inventory_candidates.append(candidate)
 497
 498             # Ask conductor-data for one or more candidates.
 499             ctxt = {
 500                 "plan_id": self._plan_id,
 501                 "plan_name": self._plan_name,
 502                 "keyspace": self.conf.keyspace,
 503             }
 504             args = {
 505                 "demands": {
 506                     name: requirements,
 507                 },
 508                 "plan_info":{
 509                     "plan_id": self._plan_id,
 510                     "plan_name": self._plan_name
 511                 },
 512                 "triage_translator_data": self.triageTranslatorData.__dict__
 513
 514             }
 515
 516             # Check if required_candidate and excluded candidate
 517             # are mutually exclusive.
 518             for requirement in requirements:
 519                 required_candidates = requirement.get("required_candidates")
 520                 excluded_candidates = requirement.get("excluded_candidates")
 521                 if (required_candidates and
 522                     excluded_candidates and
 523                     set(map(lambda entry: entry['candidate_id'],
 524                             required_candidates))
 525                     & set(map(lambda entry: entry['candidate_id'],
 526                               excluded_candidates))):
 527                     raise TranslatorException(
 528                         "Required candidate list and excluded candidate"
 529                         " list are not mutually exclusive for demand"
 530                         " {}".format(name)
 531                     )
 532             response = self.data_service.call(
 533                 ctxt=ctxt,
 534                 method="resolve_demands",
 535                 args=args)
 536
 537             resolved_demands = \
 538                 response and response.get('resolved_demands')
 539             triage_data_trans = \
 540                 response and response.get('trans')
 541
 542             required_candidates = resolved_demands \
 543                 .get('required_candidates')
 544             if not resolved_demands:
 545                 self.triageTranslator.thefinalCallTrans(triage_data_trans)
 546                 raise TranslatorException(
 547                     "Unable to resolve inventory "
 548                     "candidates for demand {}"
 549                     .format(name)
 550                 )
 551             resolved_candidates = resolved_demands.get(name)
 552             for candidate in resolved_candidates:
 553                 inventory_candidates.append(candidate)
 554             if len(inventory_candidates) < 1:
 555                 if not required_candidates:
 556                     self.triageTranslator.thefinalCallTrans(triage_data_trans)
 557                     raise TranslatorException(
 558                         "Unable to find any candidate for "
 559                         "demand {}".format(name)
 560                     )
 561                 else:
 562                     self.triageTranslator.thefinalCallTrans(triage_data_trans)
 563                     raise TranslatorException(
 564                         "Unable to find any required "
 565                         "candidate for demand {}"
 566                         .format(name)
 567                     )
 568             parsed[name] = {
 569                 "candidates": inventory_candidates,
 570             }
 571         self.triageTranslator.thefinalCallTrans(triage_data_trans)
 572         return parsed
 573
 574     def validate_hpa_constraints(self, req_prop, value):
 575         for para in value.get(req_prop):
 576             # Make sure there is at least one
 577             # set of id, type, directives and flavorProperties
 578             if not para.get('id') \
 579                     or not para.get('type') \
 580                     or not para.get('directives') \
 581                     or not para.get('flavorProperties') \
 582                     or para.get('id') == '' \
 583                     or para.get('type') == '' \
 584                     or not isinstance(para.get('directives'), list) \
 585                     or para.get('flavorProperties') == '':
 586                 raise TranslatorException(
 587                     "HPA requirements need at least "
 588                     "one set of id, type, directives and flavorProperties"
 589                 )
 590             for feature in para.get('flavorProperties'):
 591                 if type(feature) is not dict:
 592                     raise TranslatorException("HPA feature must be a dict")
 593                 # process mandatory parameter
 594                 hpa_mandatory = set(HPA_FEATURES).difference(feature.keys())
 595                 if bool(hpa_mandatory):
 596                     raise TranslatorException(
 597                         "Lack of compulsory elements inside HPA feature")
 598                 # process optional parameter
 599                 hpa_optional = set(feature.keys()).difference(HPA_FEATURES)
 600                 if hpa_optional and not hpa_optional.issubset(HPA_OPTIONAL):
 601                     raise TranslatorException(
 602                         "Got unrecognized elements inside HPA feature")
 603                 if feature.get('mandatory') == 'False' and not feature.get(
 604                         'score'):
 605                     raise TranslatorException(
 606                         "Score needs to be present if mandatory is False")
 607
 608                 for attr in feature.get('hpa-feature-attributes'):
 609                     if type(attr) is not dict:
 610                         raise TranslatorException(
 611                             "HPA feature attributes must be a dict")
 612
 613                     # process mandatory hpa attribute parameter
 614                     hpa_attr_mandatory = set(HPA_ATTRIBUTES).difference(
 615                         attr.keys())
 616                     if bool(hpa_attr_mandatory):
 617                         raise TranslatorException(
 618                             "Lack of compulsory elements inside HPA "
 619                             "feature attributes")
 620                     # process optional hpa attribute parameter
 621                     hpa_attr_optional = set(attr.keys()).difference(
 622                         HPA_ATTRIBUTES)
 623                     if hpa_attr_optional and not hpa_attr_optional.issubset(
 624                             HPA_ATTRIBUTES_OPTIONAL):
 625                         raise TranslatorException(
 626                             "Invalid attributes '{}' found inside HPA "
 627                             "feature attributes".format(hpa_attr_optional))
 628
 629     def parse_constraints(self, constraints):
 630         """Validate/prepare constraints for use by the solver."""
 631         if not isinstance(constraints, dict):
 632             raise TranslatorException("Constraints must be provided in "
 633                                       "dictionary form")
 634
 635         # Look at each constraint. Properties must exist, even if empty.
 636         constraints_copy = copy.deepcopy(constraints)
 637
 638         parsed = {}
 639         for name, constraint in constraints_copy.items():
 640
 641             if not constraint.get('properties'):
 642                 constraint['properties'] = {}
 643
 644             constraint_type = constraint.get('type')
 645             constraint_def = CONSTRAINTS.get(constraint_type)
 646
 647             # Is it a supported type?
 648             if constraint_type not in CONSTRAINTS:
 649                 raise TranslatorException(
 650                     "Unsupported type '{}' found in constraint "
 651                     "named '{}'".format(constraint_type, name))
 652
 653             # Now walk through the constraint's content
 654             for key, value in constraint.items():
 655                 # Must be a supported key
 656                 if key not in CONSTRAINT_KEYS:
 657                     raise TranslatorException(
 658                         "Invalid key '{}' found in constraint "
 659                         "named '{}'".format(key, name))
 660
 661                 # For properties ...
 662                 if key == 'properties':
 663                     # Make sure all required properties are present
 664                     required = constraint_def.get('required', [])
 665                     for req_prop in required:
 666                         if req_prop not in list(value.keys()):
 667                             raise TranslatorException(
 668                                 "Required property '{}' not found in "
 669                                 "constraint named '{}'".format(
 670                                     req_prop, name))
 671                         if not value.get(req_prop) \
 672                                 or value.get(req_prop) == '':
 673                             raise TranslatorException(
 674                                 "No value specified for property '{}' in "
 675                                 "constraint named '{}'".format(
 676                                     req_prop, name))
 677                             # For HPA constraints
 678                         if constraint_type == 'hpa':
 679                             self.validate_hpa_constraints(req_prop, value)
 680
 681                     # Make sure there are no unknown properties
 682                     optional = constraint_def.get('optional', [])
 683                     for prop_name in list(value.keys()):
 684                         if prop_name not in required + optional:
 685                             raise TranslatorException(
 686                                 "Unknown property '{}' in "
 687                                 "constraint named '{}'".format(
 688                                     prop_name, name))
 689
 690                     # If a property has a controlled vocabulary, make
 691                     # sure its value is one of the allowed ones.
 692                     allowed = constraint_def.get('allowed', {})
 693                     for prop_name, allowed_values in allowed.items():
 694                         if prop_name in list(value.keys()):
 695                             prop_value = value.get(prop_name, '')
 696                             if prop_value not in allowed_values:
 697                                 raise TranslatorException(
 698                                     "Property '{}' value '{}' unsupported in "
 699                                     "constraint named '{}' (must be one of "
 700                                     "{})".format(prop_name, prop_value,
 701                                                  name, allowed_values))
 702
 703                     # Break all threshold-formatted values into parts
 704                     thresholds = constraint_def.get('thresholds', {})
 705                     for thr_prop, base_units in thresholds.items():
 706                         if thr_prop in list(value.keys()):
 707                             expression = value.get(thr_prop)
 708                             thr = threshold.Threshold(expression, base_units)
 709                             value[thr_prop] = thr.parts
 710
 711             # We already know we have one or more demands due to
 712             # validate_components(). We still need to coerce the demands
 713             # into a list in case only one demand was provided.
 714             constraint_demands = constraint.get('demands')
 715             if isinstance(constraint_demands, six.string_types):
 716                 constraint['demands'] = [constraint_demands]
 717
 718             # Either split the constraint into parts, one per demand,
 719             # or use it as-is
 720             if constraint_def.get('split'):
 721                 for demand in constraint.get('demands', []):
 722                     constraint_demand = name + '_' + demand
 723                     parsed[constraint_demand] = copy.deepcopy(constraint)
 724                     parsed[constraint_demand]['name'] = name
 725                     parsed[constraint_demand]['demands'] = demand
 726             else:
 727                 parsed[name] = copy.deepcopy(constraint)
 728                 parsed[name]['name'] = name
 729
 730         return parsed
 731
 732     def parse_optimization(self, optimization):
 733         """Validate/prepare optimization for use by the solver."""
 734
 735         # WARNING: The template format for optimization is generalized,
 736         # however the solver is very particular about the expected
 737         # goal, functions, and operands. Therefore, for the time being,
 738         # we are choosing to be highly conservative in what we accept
 739         # at the template level. Once the solver can handle the more
 740         # general form, we can make the translation pass in this
 741         # essentially pre-parsed formula unchanged, or we may allow
 742         # optimizations to be written in algebraic form and pre-parsed
 743         # with antlr4-python2-runtime. (jdandrea 1 Dec 2016)
 744
 745         if not optimization:
 746             LOG.debug("No objective function or "
 747                       "optimzation provided in the template")
 748             return
 749
 750         optimization_copy = copy.deepcopy(optimization)
 751         parsed = {
 752             "goal": "min",
 753             "operation": "sum",
 754             "operands": [],
 755         }
 756
 757         if type(optimization_copy) is not dict:
 758             raise TranslatorException("Optimization must be a dictionary.")
 759
 760         goals = list(optimization_copy.keys())
 761         if goals != ['minimize']:
 762             raise TranslatorException(
 763                 "Optimization must contain a single goal of 'minimize'.")
 764
 765         funcs = list(optimization_copy['minimize'].keys())
 766         if funcs != ['sum']:
 767             raise TranslatorException(
 768                 "Optimization goal 'minimize' must "
 769                 "contain a single function of 'sum'.")
 770         operands = optimization_copy['minimize']['sum']
 771         if type(operands) is not list:
 772             # or len(operands) != 2:
 773             raise TranslatorException(
 774                 "Optimization goal 'minimize', function 'sum' "
 775                 "must be a list of exactly two operands.")
 776
 777         def get_latency_between_args(operand):
 778             args = operand.get('latency_between')
 779             if type(args) is not list and len(args) != 2:
 780                 raise TranslatorException(
 781                     "Optimization 'latency_between' arguments must "
 782                     "be a list of length two.")
 783
 784             got_demand = False
 785             got_location = False
 786             for arg in args:
 787                 if not got_demand and arg in list(self._demands.keys()):
 788                     got_demand = True
 789                 if not got_location and arg in list(self._locations.keys()):
 790                     got_location = True
 791             if not got_demand or not got_location:
 792                 raise TranslatorException(
 793                     "Optimization 'latency_between' arguments {} must "
 794                     "include one valid demand name and one valid "
 795                     "location name.".format(args))
 796
 797             return args
 798
 799         def get_distance_between_args(operand):
 800             args = operand.get('distance_between')
 801             if type(args) is not list and len(args) != 2:
 802                 raise TranslatorException(
 803                     "Optimization 'distance_between' arguments must "
 804                     "be a list of length two.")
 805
 806             got_demand = False
 807             got_location = False
 808             for arg in args:
 809                 if not got_demand and arg in list(self._demands.keys()):
 810                     got_demand = True
 811                 if not got_location and arg in list(self._locations.keys()):
 812                     got_location = True
 813             if not got_demand or not got_location:
 814                 raise TranslatorException(
 815                     "Optimization 'distance_between' arguments {} must "
 816                     "include one valid demand name and one valid "
 817                     "location name.".format(args))
 818
 819             return args
 820
 821         for operand in operands:
 822             weight = 1.0
 823             args = None
 824             nested = False
 825
 826             if list(operand.keys()) == ['distance_between']:
 827                 # Value must be a list of length 2 with one
 828                 # location and one demand
 829                 function = 'distance_between'
 830                 args = get_distance_between_args(operand)
 831
 832             elif list(operand.keys()) == ['product']:
 833                 for product_op in operand['product']:
 834                     if threshold.is_number(product_op):
 835                         weight = product_op
 836                     elif isinstance(product_op, dict):
 837                         if list(product_op.keys()) == ['latency_between']:
 838                             function = 'latency_between'
 839                             args = get_latency_between_args(product_op)
 840                         elif list(product_op.keys()) == ['distance_between']:
 841                             function = 'distance_between'
 842                             args = get_distance_between_args(product_op)
 843                         elif list(product_op.keys()) == ['aic_version']:
 844                             function = 'aic_version'
 845                             args = product_op.get('aic_version')
 846                         elif list(product_op.keys()) == ['hpa_score']:
 847                             function = 'hpa_score'
 848                             args = product_op.get('hpa_score')
 849                             if not self.is_hpa_policy_exists(args):
 850                                 raise TranslatorException(
 851                                     "HPA Score Optimization must include a "
 852                                     "HPA Policy constraint ")
 853                         elif list(product_op.keys()) == ['sum']:
 854                             nested = True
 855                             nested_operands = product_op.get('sum')
 856                             for nested_operand in nested_operands:
 857                                 if list(nested_operand.keys()) == ['product']:
 858                                     nested_weight = weight
 859                                     for nested_product_op in nested_operand['product']:
 860                                         if threshold.is_number(nested_product_op):
 861                                             nested_weight = nested_weight * int(nested_product_op)
 862                                         elif isinstance(nested_product_op, dict):
 863                                             if list(nested_product_op.keys()) == ['latency_between']:
 864                                                 function = 'latency_between'
 865                                                 args = get_latency_between_args(nested_product_op)
 866                                             elif list(nested_product_op.keys()) == ['distance_between']:
 867                                                 function = 'distance_between'
 868                                                 args = get_distance_between_args(nested_product_op)
 869                                     parsed['operands'].append(
 870                                         {
 871                                             "operation": "product",
 872                                             "weight": nested_weight,
 873                                             "function": function,
 874                                             "function_param": args,
 875                                         }
 876                                     )
 877
 878                 if not args:
 879                     raise TranslatorException(
 880                         "Optimization products must include at least "
 881                         "one 'distance_between' function call and "
 882                         "one optional number to be used as a weight.")
 883
 884             # We now have our weight/function_param.
 885             if not nested:
 886                 parsed['operands'].append(
 887                     {
 888                         "operation": "product",
 889                         "weight": weight,
 890                         "function": function,
 891                         "function_param": args,
 892                     }
 893                 )
 894         return parsed
 895
 896     def is_hpa_policy_exists(self, demand_list):
 897         # Check if a HPA constraint exist for the demands in the demand list.
 898         constraints_copy = copy.deepcopy(self._constraints)
 899         for demand in demand_list:
 900             for name, constraint in constraints_copy.items():
 901                 constraint_type = constraint.get('type')
 902                 if constraint_type == 'hpa':
 903                     hpa_demands = constraint.get('demands')
 904                     if demand in hpa_demands:
 905                         return True
 906         return False
 907
 908     def parse_reservations(self, reservations):
 909         demands = self._demands
 910         if not isinstance(reservations, dict):
 911             raise TranslatorException("Reservations must be provided in "
 912                                       "dictionary form")
 913         parsed = {}
 914         if reservations:
 915             parsed['counter'] = 0
 916             parsed['demands'] = {}
 917
 918         for key, value in reservations.items():
 919
 920             if key == "service_model":
 921                 parsed['service_model'] = value
 922
 923             elif key == "service_candidates":
 924                 for name, reservation_details in value.items():
 925                     if not reservation_details.get('properties'):
 926                         reservation_details['properties'] = {}
 927                     for demand in reservation_details.get('demands', []):
 928                         if demand in list(demands.keys()):
 929                             reservation_demand = name + '_' + demand
 930                             parsed['demands'][reservation_demand] = copy.deepcopy(reservation_details)
 931                             parsed['demands'][reservation_demand]['name'] = name
 932                             parsed['demands'][reservation_demand]['demands'] = demand
 933                         else:
 934                             raise TranslatorException("Demand {} must be provided in demands section".format(demand))
 935
 936         return parsed
 937
 938     def do_translation(self):
 939         """Perform the translation."""
 940         if not self.valid:
 941             raise TranslatorException("Can't translate an invalid template.")
 942
 943         request_type = self._parameters.get("request_type") \
 944                        or self._parameters.get("REQUEST_TYPE") \
 945                        or ""
 946
 947         self._translation = {
 948             "conductor_solver": {
 949                 "version": self._version,
 950                 "plan_id": self._plan_id,
 951                 "request_type": request_type,
 952                 "locations": self.parse_locations(self._locations),
 953                 "demands": self.parse_demands(self._demands),
 954                 "objective": self.parse_optimization(self._optmization),
 955                 "constraints": self.parse_constraints(self._constraints),
 956                 "objective": self.parse_optimization(self._optmization),
 957                 "reservations": self.parse_reservations(self._reservations),
 958             }
 959         }
 960
 961     def translate(self):
 962         """Translate the template for the solver."""
 963         self._ok = False
 964         try:
 965             self.create_components()
 966             self.validate_components()
 967             self.parse_parameters()
 968             self.do_translation()
 969             self._ok = True
 970         except Exception as exc:
 971             self._error_message = exc.args
 972
 973     @property
 974     def valid(self):
 975         """Returns True if the template has been validated."""
 976         return self._valid
 977
 978     @property
 979     def ok(self):
 980         """Returns True if the translation was successful."""
 981         return self._ok
 982
 983     @property
 984     def translation(self):
 985         """Returns the translation if it was successful."""
 986         return self._translation
 987
 988     @property
 989     def error_message(self):
 990         """Returns the last known error message."""
 991         return self._error_message
 992
 993
 994 def main():
 995     template_name = 'some_template'
 996
 997     path = os.path.abspath(conductor_root)
 998     dir_path = os.path.dirname(path)
 999
1000     # Prepare service-wide components (e.g., config)
1001     conf = service.prepare_service(
1002         [], config_files=[dir_path + '/../etc/conductor/conductor.conf'])
1003     # conf.set_override('mock', True, 'music_api')
1004
1005     t1 = threshold.Threshold("< 500 ms", "time")
1006     t2 = threshold.Threshold("= 120 mi", "distance")
1007     t3 = threshold.Threshold("160", "currency")
1008     t4 = threshold.Threshold("60-80 Gbps", "throughput")
1009     print('t1: {}\nt2: {}\nt3: {}\nt4: {}\n'.format(t1, t2, t3, t4))
1010
1011     template_file = dir_path + '/tests/data/' + template_name + '.yaml'
1012     fd = open(template_file, "r")
1013     template = yaml.load(fd)
1014
1015     trns = Translator(conf, template_name, str(uuid.uuid4()), template)
1016     trns.translate()
1017     if trns.ok:
1018         print(json.dumps(trns.translation, indent=2))
1019     else:
1020         print("TESTING - Translator Error: {}".format(trns.error_message))
1021
1022
1023 if __name__ == '__main__':
1024     main()