conductor/conductor/controller/translator.py

   1 #
   2 # -------------------------------------------------------------------------
   3 #   Copyright (c) 2015-2017 AT&T Intellectual Property
   4 #
   5 #   Licensed under the Apache License, Version 2.0 (the "License");
   6 #   you may not use this file except in compliance with the License.
   7 #   You may obtain a copy of the License at
   8 #
   9 #       http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 #   Unless required by applicable law or agreed to in writing, software
  12 #   distributed under the License is distributed on an "AS IS" BASIS,
  13 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 #   See the License for the specific language governing permissions and
  15 #   limitations under the License.
  16 #
  17 # -------------------------------------------------------------------------
  18 #
  19
  20 import copy
  21 import datetime
  22 import json
  23 import os
  24 import uuid
  25
  26 import six
  27 import yaml
  28 from conductor import __file__ as conductor_root
  29 from conductor import messaging
  30 from conductor import service
  31
  32 from conductor.common import threshold
  33 from conductor.common.music import messaging as music_messaging
  34 from conductor.data.plugins.triage_translator.triage_translator_data import TraigeTranslatorData
  35 from conductor.data.plugins.triage_translator.triage_translator import TraigeTranslator
  36 from oslo_config import cfg
  37 from oslo_log import log
  38
  39 LOG = log.getLogger(__name__)
  40
  41 CONF = cfg.CONF
  42
  43 VERSIONS = ["2016-11-01", "2017-10-10", "2018-02-01"]
  44 LOCATION_KEYS = ['latitude', 'longitude', 'host_name', 'clli_code']
  45 INVENTORY_PROVIDERS = ['aai']
  46 INVENTORY_TYPES = ['cloud', 'service', 'transport', 'vfmodule']
  47 DEFAULT_INVENTORY_PROVIDER = INVENTORY_PROVIDERS[0]
  48 CANDIDATE_KEYS = ['candidate_id', 'cost', 'inventory_type', 'location_id',
  49                   'location_type']
  50 DEMAND_KEYS = ['filtering_attributes', 'passthrough_attributes', 'candidates', 'complex', 'conflict_identifier',
  51                'customer_id', 'default_cost', 'excluded_candidates',
  52                'existing_placement', 'flavor', 'inventory_provider',
  53                'inventory_type', 'port_key', 'region', 'required_candidates',
  54                'service_id', 'service_resource_id', 'service_subscription',
  55                'service_type', 'subdivision', 'unique', 'vlan_key']
  56 CONSTRAINT_KEYS = ['type', 'demands', 'properties']
  57 CONSTRAINTS = {
  58     # constraint_type: {
  59     #   split: split into individual constraints, one per demand
  60     #   required: list of required property names,
  61     #   optional: list of optional property names,
  62     #   thresholds: dict of property/base-unit pairs for threshold parsing
  63     #   allowed: dict of keys and allowed values (if controlled vocab);
  64     #            only use this for Conductor-controlled values!
  65     # }
  66     'attribute': {
  67         'split': True,
  68         'required': ['evaluate'],
  69     },
  70     'distance_between_demands': {
  71         'required': ['distance'],
  72         'thresholds': {
  73             'distance': 'distance'
  74         },
  75     },
  76     'distance_to_location': {
  77         'split': True,
  78         'required': ['distance', 'location'],
  79         'thresholds': {
  80             'distance': 'distance'
  81         },
  82     },
  83     'instance_fit': {
  84         'split': True,
  85         'required': ['controller'],
  86         'optional': ['request'],
  87     },
  88     'inventory_group': {},
  89     'region_fit': {
  90         'split': True,
  91         'required': ['controller'],
  92         'optional': ['request'],
  93     },
  94     'zone': {
  95         'required': ['qualifier', 'category'],
  96         'optional': ['location'],
  97         'allowed': {'qualifier': ['same', 'different'],
  98                     'category': ['disaster', 'region', 'complex', 'country',
  99                                  'time', 'maintenance']},
 100     },
 101     'vim_fit': {
 102         'split': True,
 103         'required': ['controller'],
 104         'optional': ['request'],
 105     },
 106     'hpa': {
 107         'split': True,
 108         'required': ['evaluate'],
 109     },
 110 }
 111 HPA_FEATURES = ['architecture', 'hpa-feature', 'hpa-feature-attributes',
 112                 'hpa-version', 'mandatory', 'directives']
 113 HPA_OPTIONAL = ['score']
 114 HPA_ATTRIBUTES = ['hpa-attribute-key', 'hpa-attribute-value', 'operator']
 115 HPA_ATTRIBUTES_OPTIONAL = ['unit']
 116
 117
 118 class TranslatorException(Exception):
 119     pass
 120
 121
 122 class Translator(object):
 123     """Template translator.
 124
 125     Takes an input template and translates it into
 126     something the solver can use. Calls the data service
 127     as needed, giving it the inventory provider as context.
 128     Presently the only inventory provider is A&AI. Others
 129     may be added in the future.
 130     """
 131
 132     def __init__(self, conf, plan_name, plan_id, template):
 133         self.conf = conf
 134         self._template = copy.deepcopy(template)
 135         self._plan_name = plan_name
 136         self._plan_id = plan_id
 137         self._translation = None
 138         self._valid = False
 139         self._ok = False
 140         self.triageTranslatorData= TraigeTranslatorData()
 141         self.triageTranslator = TraigeTranslator()
 142         # Set up the RPC service(s) we want to talk to.
 143         self.data_service = self.setup_rpc(self.conf, "data")
 144
 145     def setup_rpc(self, conf, topic):
 146         """Set up the RPC Client"""
 147         # TODO(jdandrea): Put this pattern inside music_messaging?
 148         transport = messaging.get_transport(conf=conf)
 149         target = music_messaging.Target(topic=topic)
 150         client = music_messaging.RPCClient(conf=conf,
 151                                            transport=transport,
 152                                            target=target)
 153         return client
 154
 155     def create_components(self):
 156         # TODO(jdandrea): Make deep copies so the template is untouched
 157         self._version = self._template.get("homing_template_version")
 158         self._parameters = self._template.get("parameters", {})
 159         self._locations = self._template.get("locations", {})
 160         self._demands = self._template.get("demands", {})
 161         self._constraints = self._template.get("constraints", {})
 162         self._optmization = self._template.get("optimization", {})
 163         self._reservations = self._template.get("reservation", {})
 164
 165         if isinstance(self._version, datetime.date):
 166             self._version = str(self._version)
 167
 168     def validate_components(self):
 169         """Cursory validation of template components.
 170
 171         More detailed validation happens while parsing each component.
 172         """
 173         self._valid = False
 174
 175         # Check version
 176         if self._version not in VERSIONS:
 177             raise TranslatorException(
 178                 "conductor_template_version must be one "
 179                 "of: {}".format(', '.join(VERSIONS)))
 180
 181         # Check top level structure
 182         components = {
 183             "parameters": {
 184                 "name": "Parameter",
 185                 "content": self._parameters,
 186             },
 187             "locations": {
 188                 "name": "Location",
 189                 "keys": LOCATION_KEYS,
 190                 "content": self._locations,
 191             },
 192             "demands": {
 193                 "name": "Demand",
 194                 "content": self._demands,
 195             },
 196             "constraints": {
 197                 "name": "Constraint",
 198                 "keys": CONSTRAINT_KEYS,
 199                 "content": self._constraints,
 200             },
 201             "optimization": {
 202                 "name": "Optimization",
 203                 "content": self._optmization,
 204             },
 205             "reservations": {
 206                 "name": "Reservation",
 207                 "content": self._reservations,
 208             }
 209         }
 210         for name, component in components.items():
 211             name = component.get('name')
 212             keys = component.get('keys', None)
 213             content = component.get('content')
 214
 215             if type(content) is not dict:
 216                 raise TranslatorException(
 217                     "{} section must be a dictionary".format(name))
 218             for content_name, content_def in content.items():
 219                 if not keys:
 220                     continue
 221
 222                 for key in content_def:
 223                     if key not in keys:
 224                         raise TranslatorException(
 225                             "{} {} has an invalid key {}".format(
 226                                 name, content_name, key))
 227
 228         demand_keys = list(self._demands.keys())     # Python 3 Conversion -- dict object to list object
 229         location_keys = list(self._locations.keys())     # Python 3 Conversion -- dict object to list object
 230         for constraint_name, constraint in self._constraints.items():
 231
 232             # Require a single demand (string), or a list of one or more.
 233             demands = constraint.get('demands')
 234             if isinstance(demands, six.string_types):
 235                 demands = [demands]
 236             if not isinstance(demands, list) or len(demands) < 1:
 237                 raise TranslatorException(
 238                     "Demand list for Constraint {} must be "
 239                     "a list of names or a string with one name".format(
 240                         constraint_name))
 241             if not set(demands).issubset(demand_keys + location_keys):
 242                 raise TranslatorException(
 243                     "Undefined Demand(s) {} in Constraint '{}'".format(
 244                         list(set(demands).difference(demand_keys)),
 245                         constraint_name))
 246
 247             properties = constraint.get('properties', None)
 248             if properties:
 249                 location = properties.get('location', None)
 250                 if location:
 251                     if location not in location_keys:
 252                         raise TranslatorException(
 253                             "Location {} in Constraint {} is undefined".format(
 254                                 location, constraint_name))
 255
 256         self._valid = True
 257
 258     def _parse_parameters(self, obj, path=[]):
 259         """Recursively parse all {get_param: X} occurrences
 260
 261         This modifies obj in-place. If you want to keep the original,
 262         pass in a deep copy.
 263         """
 264         # Ok to start with a string ...
 265         if isinstance(path, six.string_types):
 266             # ... but the breadcrumb trail goes in an array.
 267             path = [path]
 268
 269         # Traverse a list
 270         if type(obj) is list:
 271             for idx, val in enumerate(obj, start=0):
 272                 # Add path to the breadcrumb trail
 273                 new_path = list(path)
 274                 new_path[-1] += "[{}]".format(idx)
 275
 276                 # Look at each element.
 277                 obj[idx] = self._parse_parameters(val, new_path)
 278
 279         # Traverse a dict
 280         elif type(obj) is dict:
 281             # Did we find a "{get_param: ...}" intrinsic?
 282             if list(obj.keys()) == ['get_param']:
 283                 param_name = obj['get_param']
 284
 285                 # The parameter name must be a string.
 286                 if not isinstance(param_name, six.string_types):
 287                     path_str = ' > '.join(path)
 288                     raise TranslatorException(
 289                         "Parameter name '{}' not a string in path {}".format(
 290                             param_name, path_str))
 291
 292                 # Parameter name must be defined.
 293                 if param_name not in self._parameters:
 294                     path_str = ' > '.join(path)
 295                     raise TranslatorException(
 296                         "Parameter '{}' undefined in path {}".format(
 297                             param_name, path_str))
 298
 299                 # Return the value in place of the call.
 300                 return self._parameters.get(param_name)
 301
 302             # Not an intrinsic. Traverse as usual.
 303             for key in list(obj.keys()):
 304                 # Add path to the breadcrumb trail.
 305                 new_path = list(path)
 306                 new_path.append(key)
 307
 308                 # Look at each key/value pair.
 309                 obj[key] = self._parse_parameters(obj[key], new_path)
 310
 311         # Return whatever we have after unwinding.
 312         return obj
 313
 314     def parse_parameters(self):
 315         """Resolve all parameters references."""
 316         locations = copy.deepcopy(self._locations)
 317         self._locations = self._parse_parameters(locations, 'locations')
 318
 319         demands = copy.deepcopy(self._demands)
 320         self._demands = self._parse_parameters(demands, 'demands')
 321
 322         constraints = copy.deepcopy(self._constraints)
 323         self._constraints = self._parse_parameters(constraints, 'constraints')
 324
 325         reservations = copy.deepcopy(self._reservations)
 326         self._reservations = self._parse_parameters(reservations,
 327                                                     'reservations')
 328
 329     def parse_locations(self, locations):
 330         """Prepare the locations for use by the solver."""
 331         parsed = {}
 332         for location, args in locations.items():
 333             ctxt = {
 334                 'plan_id': self._plan_id,
 335                 'keyspace': self.conf.keyspace
 336             }
 337
 338             latitude = args.get("latitude")
 339             longitude = args.get("longitude")
 340
 341             if latitude and longitude:
 342                 resolved_location = {"latitude": latitude, "longitude": longitude}
 343             else:
 344                 # ctxt = {}
 345                 response = self.data_service.call(
 346                     ctxt=ctxt,
 347                     method="resolve_location",
 348                     args=args)
 349
 350                 resolved_location = \
 351                     response and response.get('resolved_location')
 352             if not resolved_location:
 353                 raise TranslatorException(
 354                     "Unable to resolve location {}".format(location)
 355                 )
 356             parsed[location] = resolved_location
 357         return parsed
 358
 359     def parse_demands(self, demands):
 360         """Validate/prepare demands for use by the solver."""
 361         if type(demands) is not dict:
 362             raise TranslatorException("Demands must be provided in "
 363                                       "dictionary form")
 364
 365         # Look at each demand
 366         demands_copy = copy.deepcopy(demands)
 367         parsed = {}
 368         for name, requirements in demands_copy.items():
 369             inventory_candidates = []
 370             for requirement in requirements:
 371                 for key in requirement:
 372                     if key not in DEMAND_KEYS:
 373                         raise TranslatorException(
 374                             "Demand {} has an invalid key {}".format(
 375                                 requirement, key))
 376
 377                 if 'candidates' in requirement:
 378                     # Candidates *must* specify an inventory provider
 379                     provider = requirement.get("inventory_provider")
 380                     if provider and provider not in INVENTORY_PROVIDERS:
 381                         raise TranslatorException(
 382                             "Unsupported inventory provider {} "
 383                             "in demand {}".format(provider, name))
 384                     else:
 385                         provider = DEFAULT_INVENTORY_PROVIDER
 386
 387                     # Check each candidate
 388                     for candidate in requirement.get('candidates'):
 389                         # Must be a dictionary
 390                         if type(candidate) is not dict:
 391                             raise TranslatorException(
 392                                 "Candidate found in demand {} that is "
 393                                 "not a dictionary".format(name))
 394
 395                         # Must have only supported keys
 396                         for key in list(candidate.keys()):
 397                             if key not in CANDIDATE_KEYS:
 398                                 raise TranslatorException(
 399                                     "Candidate with invalid key {} found "
 400                                     "in demand {}".format(key, name)
 401                                 )
 402
 403                         # TODO(jdandrea): Check required/optional keys
 404
 405                         # Set the inventory provider if not already
 406                         candidate['inventory_provider'] = \
 407                             candidate.get('inventory_provider', provider)
 408
 409                         # Set cost if not already (default cost is 0?)
 410                         candidate['cost'] = candidate.get('cost', 0)
 411
 412                         # Add to our list of parsed candidates
 413                         inventory_candidates.append(candidate)
 414
 415                 # candidates are specified through inventory providers
 416                 # Do the basic sanity checks for inputs
 417                 else:
 418                     # inventory provider MUST be specified
 419                     provider = requirement.get("inventory_provider")
 420                     if not provider:
 421                         raise TranslatorException(
 422                             "Inventory provider not specified "
 423                             "in demand {}".format(name)
 424                         )
 425                     elif provider and provider not in INVENTORY_PROVIDERS:
 426                         raise TranslatorException(
 427                             "Unsupported inventory provider {} "
 428                             "in demand {}".format(provider, name)
 429                         )
 430                     else:
 431                         provider = DEFAULT_INVENTORY_PROVIDER
 432                         requirement['provider'] = provider
 433
 434                     # inventory type MUST be specified
 435                     inventory_type = requirement.get('inventory_type')
 436                     if not inventory_type or inventory_type == '':
 437                         raise TranslatorException(
 438                             "Inventory type not specified for "
 439                             "demand {}".format(name)
 440                         )
 441                     if inventory_type and \
 442                             inventory_type not in INVENTORY_TYPES:
 443                         raise TranslatorException(
 444                             "Unknown inventory type {} specified for "
 445                             "demand {}".format(inventory_type, name)
 446                         )
 447
 448                     # For service and vfmodule inventories, customer_id and
 449                     # service_type MUST be specified
 450                     if inventory_type == 'service' or inventory_type == 'vfmodule':
 451                         filtering_attributes = requirement.get('filtering_attributes')
 452
 453                         if filtering_attributes:
 454                             customer_id = filtering_attributes.get('customer-id')
 455                             global_customer_id = filtering_attributes.get('global-customer-id')
 456                             if global_customer_id:
 457                                 customer_id = global_customer_id
 458                         else:
 459                             # for backward compatibility
 460                             customer_id = requirement.get('customer_id')
 461                             service_type = requirement.get('service_type')
 462
 463                         if not customer_id:
 464                             raise TranslatorException(
 465                                 "Customer ID not specified for "
 466                                 "demand {}".format(name)
 467                             )
 468                         if not filtering_attributes and not service_type:
 469                             raise TranslatorException(
 470                                 "Service Type not specified for "
 471                                 "demand {}".format(name)
 472                             )
 473
 474                 # TODO(jdandrea): Check required/optional keys for requirement
 475                 # elif 'inventory_type' in requirement:
 476                 #     # For now this is just a stand-in candidate
 477                 #     candidate = {
 478                 #         'inventory_provider':
 479                 #             requirement.get('inventory_provider',
 480                 #                             DEFAULT_INVENTORY_PROVIDER),
 481                 #         'inventory_type':
 482                 #             requirement.get('inventory_type', ''),
 483                 #         'candidate_id': '',
 484                 #         'location_id': '',
 485                 #         'location_type': '',
 486                 #         'cost': 0,
 487                 #     }
 488                 #
 489                 #     # Add to our list of parsed candidates
 490                 #     inventory_candidates.append(candidate)
 491
 492             # Ask conductor-data for one or more candidates.
 493             ctxt = {
 494                 "plan_id": self._plan_id,
 495                 "plan_name": self._plan_name,
 496                 "keyspace": self.conf.keyspace,
 497             }
 498             args = {
 499                 "demands": {
 500                     name: requirements,
 501                 },
 502                 "plan_info":{
 503                     "plan_id": self._plan_id,
 504                     "plan_name": self._plan_name
 505                 },
 506                 "triage_translator_data": self.triageTranslatorData.__dict__
 507
 508             }
 509
 510             # Check if required_candidate and excluded candidate
 511             # are mutually exclusive.
 512             for requirement in requirements:
 513                 required_candidates = requirement.get("required_candidates")
 514                 excluded_candidates = requirement.get("excluded_candidates")
 515                 if (required_candidates and
 516                     excluded_candidates and
 517                     set(map(lambda entry: entry['candidate_id'],
 518                             required_candidates))
 519                     & set(map(lambda entry: entry['candidate_id'],
 520                               excluded_candidates))):
 521                     raise TranslatorException(
 522                         "Required candidate list and excluded candidate"
 523                         " list are not mutually exclusive for demand"
 524                         " {}".format(name)
 525                     )
 526             response = self.data_service.call(
 527                 ctxt=ctxt,
 528                 method="resolve_demands",
 529                 args=args)
 530
 531             resolved_demands = \
 532                 response and response.get('resolved_demands')
 533             triage_data_trans = \
 534                 response and response.get('trans')
 535
 536             required_candidates = resolved_demands \
 537                 .get('required_candidates')
 538             if not resolved_demands:
 539                 self.triageTranslator.thefinalCallTrans(triage_data_trans)
 540                 raise TranslatorException(
 541                     "Unable to resolve inventory "
 542                     "candidates for demand {}"
 543                     .format(name)
 544                 )
 545             resolved_candidates = resolved_demands.get(name)
 546             for candidate in resolved_candidates:
 547                 inventory_candidates.append(candidate)
 548             if len(inventory_candidates) < 1:
 549                 if not required_candidates:
 550                     self.triageTranslator.thefinalCallTrans(triage_data_trans)
 551                     raise TranslatorException(
 552                         "Unable to find any candidate for "
 553                         "demand {}".format(name)
 554                     )
 555                 else:
 556                     self.triageTranslator.thefinalCallTrans(triage_data_trans)
 557                     raise TranslatorException(
 558                         "Unable to find any required "
 559                         "candidate for demand {}"
 560                         .format(name)
 561                     )
 562             parsed[name] = {
 563                 "candidates": inventory_candidates,
 564             }
 565         self.triageTranslator.thefinalCallTrans(triage_data_trans)
 566         return parsed
 567
 568     def validate_hpa_constraints(self, req_prop, value):
 569         for para in value.get(req_prop):
 570             # Make sure there is at least one
 571             # set of id, type, directives and flavorProperties
 572             if not para.get('id') \
 573                     or not para.get('type') \
 574                     or not para.get('directives') \
 575                     or not para.get('flavorProperties') \
 576                     or para.get('id') == '' \
 577                     or para.get('type') == '' \
 578                     or not isinstance(para.get('directives'), list) \
 579                     or para.get('flavorProperties') == '':
 580                 raise TranslatorException(
 581                     "HPA requirements need at least "
 582                     "one set of id, type, directives and flavorProperties"
 583                 )
 584             for feature in para.get('flavorProperties'):
 585                 if type(feature) is not dict:
 586                     raise TranslatorException("HPA feature must be a dict")
 587                 # process mandatory parameter
 588                 hpa_mandatory = set(HPA_FEATURES).difference(feature.keys())
 589                 if bool(hpa_mandatory):
 590                     raise TranslatorException(
 591                         "Lack of compulsory elements inside HPA feature")
 592                 # process optional parameter
 593                 hpa_optional = set(feature.keys()).difference(HPA_FEATURES)
 594                 if hpa_optional and not hpa_optional.issubset(HPA_OPTIONAL):
 595                     raise TranslatorException(
 596                         "Got unrecognized elements inside HPA feature")
 597                 if feature.get('mandatory') == 'False' and not feature.get(
 598                         'score'):
 599                     raise TranslatorException(
 600                         "Score needs to be present if mandatory is False")
 601
 602                 for attr in feature.get('hpa-feature-attributes'):
 603                     if type(attr) is not dict:
 604                         raise TranslatorException(
 605                             "HPA feature attributes must be a dict")
 606
 607                     # process mandatory hpa attribute parameter
 608                     hpa_attr_mandatory = set(HPA_ATTRIBUTES).difference(
 609                         attr.keys())
 610                     if bool(hpa_attr_mandatory):
 611                         raise TranslatorException(
 612                             "Lack of compulsory elements inside HPA "
 613                             "feature attributes")
 614                     # process optional hpa attribute parameter
 615                     hpa_attr_optional = set(attr.keys()).difference(
 616                         HPA_ATTRIBUTES)
 617                     if hpa_attr_optional and not hpa_attr_optional.issubset(
 618                             HPA_ATTRIBUTES_OPTIONAL):
 619                         raise TranslatorException(
 620                             "Invalid attributes '{}' found inside HPA "
 621                             "feature attributes".format(hpa_attr_optional))
 622
 623     def parse_constraints(self, constraints):
 624         """Validate/prepare constraints for use by the solver."""
 625         if not isinstance(constraints, dict):
 626             raise TranslatorException("Constraints must be provided in "
 627                                       "dictionary form")
 628
 629         # Look at each constraint. Properties must exist, even if empty.
 630         constraints_copy = copy.deepcopy(constraints)
 631
 632         parsed = {}
 633         for name, constraint in constraints_copy.items():
 634
 635             if not constraint.get('properties'):
 636                 constraint['properties'] = {}
 637
 638             constraint_type = constraint.get('type')
 639             constraint_def = CONSTRAINTS.get(constraint_type)
 640
 641             # Is it a supported type?
 642             if constraint_type not in CONSTRAINTS:
 643                 raise TranslatorException(
 644                     "Unsupported type '{}' found in constraint "
 645                     "named '{}'".format(constraint_type, name))
 646
 647             # Now walk through the constraint's content
 648             for key, value in constraint.items():
 649                 # Must be a supported key
 650                 if key not in CONSTRAINT_KEYS:
 651                     raise TranslatorException(
 652                         "Invalid key '{}' found in constraint "
 653                         "named '{}'".format(key, name))
 654
 655                 # For properties ...
 656                 if key == 'properties':
 657                     # Make sure all required properties are present
 658                     required = constraint_def.get('required', [])
 659                     for req_prop in required:
 660                         if req_prop not in list(value.keys()):
 661                             raise TranslatorException(
 662                                 "Required property '{}' not found in "
 663                                 "constraint named '{}'".format(
 664                                     req_prop, name))
 665                         if not value.get(req_prop) \
 666                                 or value.get(req_prop) == '':
 667                             raise TranslatorException(
 668                                 "No value specified for property '{}' in "
 669                                 "constraint named '{}'".format(
 670                                     req_prop, name))
 671                             # For HPA constraints
 672                         if constraint_type == 'hpa':
 673                             self.validate_hpa_constraints(req_prop, value)
 674
 675                     # Make sure there are no unknown properties
 676                     optional = constraint_def.get('optional', [])
 677                     for prop_name in list(value.keys()):
 678                         if prop_name not in required + optional:
 679                             raise TranslatorException(
 680                                 "Unknown property '{}' in "
 681                                 "constraint named '{}'".format(
 682                                     prop_name, name))
 683
 684                     # If a property has a controlled vocabulary, make
 685                     # sure its value is one of the allowed ones.
 686                     allowed = constraint_def.get('allowed', {})
 687                     for prop_name, allowed_values in allowed.items():
 688                         if prop_name in list(value.keys()):
 689                             prop_value = value.get(prop_name, '')
 690                             if prop_value not in allowed_values:
 691                                 raise TranslatorException(
 692                                     "Property '{}' value '{}' unsupported in "
 693                                     "constraint named '{}' (must be one of "
 694                                     "{})".format(prop_name, prop_value,
 695                                                  name, allowed_values))
 696
 697                     # Break all threshold-formatted values into parts
 698                     thresholds = constraint_def.get('thresholds', {})
 699                     for thr_prop, base_units in thresholds.items():
 700                         if thr_prop in list(value.keys()):
 701                             expression = value.get(thr_prop)
 702                             thr = threshold.Threshold(expression, base_units)
 703                             value[thr_prop] = thr.parts
 704
 705             # We already know we have one or more demands due to
 706             # validate_components(). We still need to coerce the demands
 707             # into a list in case only one demand was provided.
 708             constraint_demands = constraint.get('demands')
 709             if isinstance(constraint_demands, six.string_types):
 710                 constraint['demands'] = [constraint_demands]
 711
 712             # Either split the constraint into parts, one per demand,
 713             # or use it as-is
 714             if constraint_def.get('split'):
 715                 for demand in constraint.get('demands', []):
 716                     constraint_demand = name + '_' + demand
 717                     parsed[constraint_demand] = copy.deepcopy(constraint)
 718                     parsed[constraint_demand]['name'] = name
 719                     parsed[constraint_demand]['demands'] = demand
 720             else:
 721                 parsed[name] = copy.deepcopy(constraint)
 722                 parsed[name]['name'] = name
 723
 724         return parsed
 725
 726     def parse_optimization(self, optimization):
 727         """Validate/prepare optimization for use by the solver."""
 728
 729         # WARNING: The template format for optimization is generalized,
 730         # however the solver is very particular about the expected
 731         # goal, functions, and operands. Therefore, for the time being,
 732         # we are choosing to be highly conservative in what we accept
 733         # at the template level. Once the solver can handle the more
 734         # general form, we can make the translation pass in this
 735         # essentially pre-parsed formula unchanged, or we may allow
 736         # optimizations to be written in algebraic form and pre-parsed
 737         # with antlr4-python2-runtime. (jdandrea 1 Dec 2016)
 738
 739         if not optimization:
 740             LOG.debug("No objective function or "
 741                       "optimzation provided in the template")
 742             return
 743
 744         optimization_copy = copy.deepcopy(optimization)
 745         parsed = {
 746             "goal": "min",
 747             "operation": "sum",
 748             "operands": [],
 749         }
 750
 751         if type(optimization_copy) is not dict:
 752             raise TranslatorException("Optimization must be a dictionary.")
 753
 754         goals = list(optimization_copy.keys())
 755         if goals != ['minimize']:
 756             raise TranslatorException(
 757                 "Optimization must contain a single goal of 'minimize'.")
 758
 759         funcs = list(optimization_copy['minimize'].keys())
 760         if funcs != ['sum']:
 761             raise TranslatorException(
 762                 "Optimization goal 'minimize' must "
 763                 "contain a single function of 'sum'.")
 764
 765         operands = optimization_copy['minimize']['sum']
 766         if type(operands) is not list:
 767             # or len(operands) != 2:
 768             raise TranslatorException(
 769                 "Optimization goal 'minimize', function 'sum' "
 770                 "must be a list of exactly two operands.")
 771
 772         def get_latency_between_args(operand):
 773             args = operand.get('latency_between')
 774             if type(args) is not list and len(args) != 2:
 775                 raise TranslatorException(
 776                     "Optimization 'latency_between' arguments must "
 777                     "be a list of length two.")
 778
 779             got_demand = False
 780             got_location = False
 781             for arg in args:
 782                 if not got_demand and arg in list(self._demands.keys()):
 783                     got_demand = True
 784                 if not got_location and arg in list(self._locations.keys()):
 785                     got_location = True
 786             if not got_demand or not got_location:
 787                 raise TranslatorException(
 788                     "Optimization 'latency_between' arguments {} must "
 789                     "include one valid demand name and one valid "
 790                     "location name.".format(args))
 791
 792             return args
 793
 794         def get_distance_between_args(operand):
 795             args = operand.get('distance_between')
 796             if type(args) is not list and len(args) != 2:
 797                 raise TranslatorException(
 798                     "Optimization 'distance_between' arguments must "
 799                     "be a list of length two.")
 800
 801             got_demand = False
 802             got_location = False
 803             for arg in args:
 804                 if not got_demand and arg in list(self._demands.keys()):
 805                     got_demand = True
 806                 if not got_location and arg in list(self._locations.keys()):
 807                     got_location = True
 808             if not got_demand or not got_location:
 809                 raise TranslatorException(
 810                     "Optimization 'distance_between' arguments {} must "
 811                     "include one valid demand name and one valid "
 812                     "location name.".format(args))
 813
 814             return args
 815
 816         for operand in operands:
 817             weight = 1.0
 818             args = None
 819             nested = False
 820
 821             if list(operand.keys()) == ['distance_between']:
 822                 # Value must be a list of length 2 with one
 823                 # location and one demand
 824                 function = 'distance_between'
 825                 args = get_distance_between_args(operand)
 826
 827             elif list(operand.keys()) == ['product']:
 828                 for product_op in operand['product']:
 829                     if threshold.is_number(product_op):
 830                         weight = product_op
 831                     elif isinstance(product_op, dict):
 832                         if list(product_op.keys()) == ['latency_between']:
 833                             function = 'latency_between'
 834                             args = get_latency_between_args(product_op)
 835                         elif list(product_op.keys()) == ['distance_between']:
 836                             function = 'distance_between'
 837                             args = get_distance_between_args(product_op)
 838                         elif list(product_op.keys()) == ['aic_version']:
 839                             function = 'aic_version'
 840                             args = product_op.get('aic_version')
 841                         elif list(product_op.keys()) == ['hpa_score']:
 842                             function = 'hpa_score'
 843                             args = product_op.get('hpa_score')
 844                             if not self.is_hpa_policy_exists(args):
 845                                 raise TranslatorException(
 846                                     "HPA Score Optimization must include a "
 847                                     "HPA Policy constraint ")
 848                         elif list(product_op.keys()) == ['sum']:
 849                             nested = True
 850                             nested_operands = product_op.get('sum')
 851                             for nested_operand in nested_operands:
 852                                 if list(nested_operand.keys()) == ['product']:
 853                                     nested_weight = weight
 854                                     for nested_product_op in nested_operand['product']:
 855                                         if threshold.is_number(nested_product_op):
 856                                             nested_weight = nested_weight * int(nested_product_op)
 857                                         elif isinstance(nested_product_op, dict):
 858                                             if list(nested_product_op.keys()) == ['latency_between']:
 859                                                 function = 'latency_between'
 860                                                 args = get_latency_between_args(nested_product_op)
 861                                             elif list(nested_product_op.keys()) == ['distance_between']:
 862                                                 function = 'distance_between'
 863                                                 args = get_distance_between_args(nested_product_op)
 864                                     parsed['operands'].append(
 865                                         {
 866                                             "operation": "product",
 867                                             "weight": nested_weight,
 868                                             "function": function,
 869                                             "function_param": args,
 870                                         }
 871                                     )
 872
 873                 if not args:
 874                     raise TranslatorException(
 875                         "Optimization products must include at least "
 876                         "one 'distance_between' function call and "
 877                         "one optional number to be used as a weight.")
 878
 879             # We now have our weight/function_param.
 880             if not nested:
 881                 parsed['operands'].append(
 882                     {
 883                         "operation": "product",
 884                         "weight": weight,
 885                         "function": function,
 886                         "function_param": args,
 887                     }
 888                 )
 889         return parsed
 890
 891     def is_hpa_policy_exists(self, demand_list):
 892         # Check if a HPA constraint exist for the demands in the demand list.
 893         constraints_copy = copy.deepcopy(self._constraints)
 894         for demand in demand_list:
 895             for name, constraint in constraints_copy.items():
 896                 constraint_type = constraint.get('type')
 897                 if constraint_type == 'hpa':
 898                     hpa_demands = constraint.get('demands')
 899                     if demand in hpa_demands:
 900                         return True
 901         return False
 902
 903     def parse_reservations(self, reservations):
 904         demands = self._demands
 905         if not isinstance(reservations, dict):
 906             raise TranslatorException("Reservations must be provided in "
 907                                       "dictionary form")
 908         parsed = {}
 909         if reservations:
 910             parsed['counter'] = 0
 911             parsed['demands'] = {}
 912
 913         for key, value in reservations.items():
 914
 915             if key == "service_model":
 916                 parsed['service_model'] = value
 917
 918             elif key == "service_candidates":
 919                 for name, reservation_details in value.items():
 920                     if not reservation_details.get('properties'):
 921                         reservation_details['properties'] = {}
 922                     for demand in reservation_details.get('demands', []):
 923                         if demand in list(demands.keys()):
 924                             reservation_demand = name + '_' + demand
 925                             parsed['demands'][reservation_demand] = copy.deepcopy(reservation_details)
 926                             parsed['demands'][reservation_demand]['name'] = name
 927                             parsed['demands'][reservation_demand]['demands'] = demand
 928                         else:
 929                             raise TranslatorException("Demand {} must be provided in demands section".format(demand))
 930
 931         return parsed
 932
 933     def do_translation(self):
 934         """Perform the translation."""
 935         if not self.valid:
 936             raise TranslatorException("Can't translate an invalid template.")
 937
 938         request_type = self._parameters.get("request_type") \
 939                        or self._parameters.get("REQUEST_TYPE") \
 940                        or ""
 941
 942         self._translation = {
 943             "conductor_solver": {
 944                 "version": self._version,
 945                 "plan_id": self._plan_id,
 946                 "request_type": request_type,
 947                 "locations": self.parse_locations(self._locations),
 948                 "demands": self.parse_demands(self._demands),
 949                 "objective": self.parse_optimization(self._optmization),
 950                 "constraints": self.parse_constraints(self._constraints),
 951                 "objective": self.parse_optimization(self._optmization),
 952                 "reservations": self.parse_reservations(self._reservations),
 953             }
 954         }
 955
 956     def translate(self):
 957         """Translate the template for the solver."""
 958         self._ok = False
 959         try:
 960             self.create_components()
 961             self.validate_components()
 962             self.parse_parameters()
 963             self.do_translation()
 964             self._ok = True
 965         except Exception as exc:
 966             self._error_message = exc.args
 967
 968     @property
 969     def valid(self):
 970         """Returns True if the template has been validated."""
 971         return self._valid
 972
 973     @property
 974     def ok(self):
 975         """Returns True if the translation was successful."""
 976         return self._ok
 977
 978     @property
 979     def translation(self):
 980         """Returns the translation if it was successful."""
 981         return self._translation
 982
 983     @property
 984     def error_message(self):
 985         """Returns the last known error message."""
 986         return self._error_message
 987
 988
 989 def main():
 990     template_name = 'some_template'
 991
 992     path = os.path.abspath(conductor_root)
 993     dir_path = os.path.dirname(path)
 994
 995     # Prepare service-wide components (e.g., config)
 996     conf = service.prepare_service(
 997         [], config_files=[dir_path + '/../etc/conductor/conductor.conf'])
 998     # conf.set_override('mock', True, 'music_api')
 999
1000     t1 = threshold.Threshold("< 500 ms", "time")
1001     t2 = threshold.Threshold("= 120 mi", "distance")
1002     t3 = threshold.Threshold("160", "currency")
1003     t4 = threshold.Threshold("60-80 Gbps", "throughput")
1004     print('t1: {}\nt2: {}\nt3: {}\nt4: {}\n'.format(t1, t2, t3, t4))
1005
1006     template_file = dir_path + '/tests/data/' + template_name + '.yaml'
1007     fd = open(template_file, "r")
1008     template = yaml.load(fd)
1009
1010     trns = Translator(conf, template_name, str(uuid.uuid4()), template)
1011     trns.translate()
1012     if trns.ok:
1013         print(json.dumps(trns.translation, indent=2))
1014     else:
1015         print("TESTING - Translator Error: {}".format(trns.error_message))
1016
1017
1018 if __name__ == '__main__':
1019     main()