From f7a27497dd184da6259ea8bd87c3c704df519923 Mon Sep 17 00:00:00 2001 From: krishnaa96 Date: Sat, 15 Aug 2020 22:29:23 +0530 Subject: [PATCH] Add support to generic optimization structure Add a new template version to support the new optmization model Issue-ID: OPTFRA-730 Signed-off-by: krishnaa96 Change-Id: I286f7ae1bad0af1fac0da7e96f7274eb9518e031 --- conductor.conf | 4 + .../controller/generic_objective_translator.py | 77 +++++++++++ conductor/conductor/controller/translator.py | 113 ++++------------ conductor/conductor/controller/translator_svc.py | 52 ++++--- conductor/conductor/controller/translator_utils.py | 104 ++++++++++++++ conductor/conductor/solver/optimizer/best_first.py | 8 +- conductor/conductor/solver/optimizer/fit_first.py | 14 +- conductor/conductor/solver/optimizer/greedy.py | 5 + .../conductor/solver/request/functions/__init__.py | 26 ++++ .../solver/request/functions/attribute.py | 32 +++++ .../solver/request/functions/distance_between.py | 5 +- .../solver/request/functions/latency_between.py | 8 +- .../solver/request/functions/location_function.py | 35 +++++ .../conductor/solver/request/generic_objective.py | 78 +++++++++++ conductor/conductor/solver/request/parser.py | 58 +++----- conductor/conductor/solver/utils/utils.py | 20 ++- .../tests/unit/controller/opt_schema.json | 138 +++++++++++++++++++ .../tests/unit/controller/template_v2.json | 150 +++++++++++++++++++++ .../test_generic_objective_translator.py | 101 ++++++++++++++ .../tests/unit/controller/test_translator_svc.py | 5 +- .../solver/request/functions/test_attribute.py | 47 +++++++ .../tests/unit/solver/request/objective.json | 97 +++++++++++++ .../unit/solver/request/test_generic_objective.py | 71 ++++++++++ conductor/etc/conductor/opt_schema.json | 138 +++++++++++++++++++ conductor/requirements.txt | 1 + conductor/tox.ini | 6 +- 26 files changed, 1230 insertions(+), 163 deletions(-) create mode 100644 conductor/conductor/controller/generic_objective_translator.py create mode 100644 conductor/conductor/controller/translator_utils.py create mode 100644 conductor/conductor/solver/request/functions/attribute.py create mode 100644 conductor/conductor/solver/request/functions/location_function.py create mode 100644 conductor/conductor/solver/request/generic_objective.py create mode 100644 conductor/conductor/tests/unit/controller/opt_schema.json create mode 100644 conductor/conductor/tests/unit/controller/template_v2.json create mode 100644 conductor/conductor/tests/unit/controller/test_generic_objective_translator.py create mode 100644 conductor/conductor/tests/unit/solver/request/functions/test_attribute.py create mode 100644 conductor/conductor/tests/unit/solver/request/objective.json create mode 100644 conductor/conductor/tests/unit/solver/request/test_generic_objective.py create mode 100644 conductor/etc/conductor/opt_schema.json diff --git a/conductor.conf b/conductor.conf index 215bf6e..7b04ec7 100755 --- a/conductor.conf +++ b/conductor.conf @@ -312,6 +312,10 @@ concurrent = true # Minimum value: 1 #max_translation_counter = 1 +# JSON schema file for optimization object +# (string value) +opt_schema_file= /opt/has/conductor/etc/conductor/opt_schema.json + [data] diff --git a/conductor/conductor/controller/generic_objective_translator.py b/conductor/conductor/controller/generic_objective_translator.py new file mode 100644 index 0000000..4b4db51 --- /dev/null +++ b/conductor/conductor/controller/generic_objective_translator.py @@ -0,0 +1,77 @@ +# +# ------------------------------------------------------------------------- +# Copyright (C) 2020 Wipro Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import copy +from jsonschema import validate +from jsonschema import ValidationError +from oslo_log import log + +from conductor.controller.translator import Translator +from conductor.controller.translator_utils import OPTIMIZATION_FUNCTIONS +from conductor.controller.translator_utils import TranslatorException + +LOG = log.getLogger(__name__) + + +class GenericObjectiveTranslator(Translator): + + def __init__(self, conf, plan_name, plan_id, template, opt_schema): + super(GenericObjectiveTranslator, self).__init__(conf, plan_name, plan_id, template) + self.translator_version = 'GENERIC' + self.opt_schema = opt_schema + + def parse_optimization(self, optimization): + + if not optimization: + LOG.debug('No optimization object is provided ' + 'in the template') + return + + self.validate(optimization) + parsed = copy.deepcopy(optimization) + self.parse_functions(parsed.get('operation_function')) + return parsed + + def validate(self, optimization): + try: + validate(instance=optimization, schema=self.opt_schema) + except ValidationError as ve: + LOG.error('Optimization object is not valid') + raise TranslatorException('Optimization object is not valid. ' + 'Validation error: {}'.format(ve.message)) + + def parse_functions(self, operation_function): + operands = operation_function.get("operands") + for operand in operands: + if 'function' in operand: + function = operand.get('function') + params = operand.get('params') + parsed_params = {} + for keyword in OPTIMIZATION_FUNCTIONS.get(function): + if keyword in params: + value = params.get(keyword) + if keyword == "demand" and value not in list(self._demands.keys()): + raise TranslatorException('{} is not a valid demand name'.format(value)) + parsed_params[keyword] = value + else: + raise TranslatorException('The function {} expect the param {},' + 'but not found'.format(function, keyword)) + operand['params'] = parsed_params + elif 'operation_function' in operand: + self.parse_functions(operand.get('operation_function')) diff --git a/conductor/conductor/controller/translator.py b/conductor/conductor/controller/translator.py index 83c71ed..dc234cd 100644 --- a/conductor/conductor/controller/translator.py +++ b/conductor/conductor/controller/translator.py @@ -24,105 +24,38 @@ import json import os import uuid +from oslo_config import cfg +from oslo_log import log import six import yaml from conductor import __file__ as conductor_root from conductor.common.music import messaging as music_messaging from conductor.common import threshold +from conductor.controller.translator_utils import CANDIDATE_KEYS +from conductor.controller.translator_utils import CONSTRAINT_KEYS +from conductor.controller.translator_utils import CONSTRAINTS +from conductor.controller.translator_utils import DEFAULT_INVENTORY_PROVIDER +from conductor.controller.translator_utils import DEMAND_KEYS +from conductor.controller.translator_utils import HPA_ATTRIBUTES +from conductor.controller.translator_utils import HPA_ATTRIBUTES_OPTIONAL +from conductor.controller.translator_utils import HPA_FEATURES +from conductor.controller.translator_utils import HPA_OPTIONAL +from conductor.controller.translator_utils import INVENTORY_PROVIDERS +from conductor.controller.translator_utils import INVENTORY_TYPES +from conductor.controller.translator_utils import LOCATION_KEYS +from conductor.controller.translator_utils import TranslatorException +from conductor.controller.translator_utils import VERSIONS from conductor.data.plugins.triage_translator.triage_translator import TraigeTranslator from conductor.data.plugins.triage_translator.triage_translator_data import TraigeTranslatorData from conductor import messaging from conductor import service -from oslo_config import cfg -from oslo_log import log + LOG = log.getLogger(__name__) CONF = cfg.CONF -VERSIONS = ["2016-11-01", "2017-10-10", "2018-02-01"] -LOCATION_KEYS = ['latitude', 'longitude', 'host_name', 'clli_code'] -INVENTORY_PROVIDERS = ['aai', 'generator'] -INVENTORY_TYPES = ['cloud', 'service', 'transport', 'vfmodule', 'nssi'] -DEFAULT_INVENTORY_PROVIDER = INVENTORY_PROVIDERS[0] -CANDIDATE_KEYS = ['candidate_id', 'cost', 'inventory_type', 'location_id', - 'location_type'] -DEMAND_KEYS = ['filtering_attributes', 'passthrough_attributes', 'default_attributes', 'candidates', 'complex', - 'conflict_identifier', 'customer_id', 'default_cost', 'excluded_candidates', - 'existing_placement', 'flavor', 'inventory_provider', - 'inventory_type', 'port_key', 'region', 'required_candidates', - 'service_id', 'service_resource_id', 'service_subscription', - 'service_type', 'subdivision', 'unique', 'vlan_key'] -CONSTRAINT_KEYS = ['type', 'demands', 'properties'] -CONSTRAINTS = { - # constraint_type: { - # split: split into individual constraints, one per demand - # required: list of required property names, - # optional: list of optional property names, - # thresholds: dict of property/base-unit pairs for threshold parsing - # allowed: dict of keys and allowed values (if controlled vocab); - # only use this for Conductor-controlled values! - # } - 'attribute': { - 'split': True, - 'required': ['evaluate'], - }, - 'threshold': { - 'split': True, - 'required': ['evaluate'], - }, - 'distance_between_demands': { - 'required': ['distance'], - 'thresholds': { - 'distance': 'distance' - }, - }, - 'distance_to_location': { - 'split': True, - 'required': ['distance', 'location'], - 'thresholds': { - 'distance': 'distance' - }, - }, - 'instance_fit': { - 'split': True, - 'required': ['controller'], - 'optional': ['request'], - }, - 'inventory_group': {}, - 'region_fit': { - 'split': True, - 'required': ['controller'], - 'optional': ['request'], - }, - 'zone': { - 'required': ['qualifier', 'category'], - 'optional': ['location'], - 'allowed': {'qualifier': ['same', 'different'], - 'category': ['disaster', 'region', 'complex', 'country', - 'time', 'maintenance']}, - }, - 'vim_fit': { - 'split': True, - 'required': ['controller'], - 'optional': ['request'], - }, - 'hpa': { - 'split': True, - 'required': ['evaluate'], - }, -} -HPA_FEATURES = ['architecture', 'hpa-feature', 'hpa-feature-attributes', - 'hpa-version', 'mandatory', 'directives'] -HPA_OPTIONAL = ['score'] -HPA_ATTRIBUTES = ['hpa-attribute-key', 'hpa-attribute-value', 'operator'] -HPA_ATTRIBUTES_OPTIONAL = ['unit'] - - -class TranslatorException(Exception): - pass - class Translator(object): """Template translator. @@ -136,6 +69,7 @@ class Translator(object): def __init__(self, conf, plan_name, plan_id, template): self.conf = conf + self.translator_version = 'BASE' self._template = copy.deepcopy(template) self._plan_name = plan_name self._plan_id = plan_id @@ -178,10 +112,10 @@ class Translator(object): self._valid = False # Check version - if self._version not in VERSIONS: + if self._version not in VERSIONS.get(self.translator_version): raise TranslatorException( "conductor_template_version must be one " - "of: {}".format(', '.join(VERSIONS))) + "of: {}".format(', '.join(VERSIONS.get(self.translator_version)))) # Check top level structure components = { @@ -517,8 +451,11 @@ class Translator(object): for requirement in requirements: required_candidates = requirement.get("required_candidates") excluded_candidates = requirement.get("excluded_candidates") - if (required_candidates and excluded_candidates and set(map(lambda entry: entry['candidate_id'], - required_candidates)) + + if (required_candidates + and excluded_candidates + and set(map(lambda entry: entry['candidate_id'], + required_candidates)) & set(map(lambda entry: entry['candidate_id'], excluded_candidates))): raise TranslatorException( diff --git a/conductor/conductor/controller/translator_svc.py b/conductor/conductor/controller/translator_svc.py index 62e885b..651e4da 100644 --- a/conductor/conductor/controller/translator_svc.py +++ b/conductor/conductor/controller/translator_svc.py @@ -1,6 +1,7 @@ # # ------------------------------------------------------------------------- # Copyright (c) 2015-2017 AT&T Intellectual Property +# Copyright (C) 2020 Wipro Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,8 +18,6 @@ # ------------------------------------------------------------------------- # -import json -import os import socket import time @@ -27,12 +26,18 @@ import futurist from oslo_config import cfg from oslo_log import log +from conductor.common.config_loader import load_config_file from conductor.common.music import api from conductor.common.music import messaging as music_messaging -from conductor.controller import translator -from conductor.i18n import _LE, _LI -from conductor import messaging from conductor.common.utils import conductor_logging_util as log_util +from conductor.controller.generic_objective_translator import GenericObjectiveTranslator +from conductor.controller.translator import Translator +from conductor.controller.translator_utils import TranslatorException +from conductor.controller.translator_utils import VERSIONS +from conductor.i18n import _LE +from conductor.i18n import _LI +from conductor import messaging + LOG = log.getLogger(__name__) @@ -46,7 +51,11 @@ CONTROLLER_OPTS = [ 'Default value is 1.'), cfg.IntOpt('max_translation_counter', default=1, - min=1) + min=1), + cfg.StrOpt('opt_schema_file', + default='opt_schema.json', + help='json schema file which will be used to validate the ' + 'optimization object for the new template version'), ] CONF.register_opts(CONTROLLER_OPTS, group='controller') @@ -64,7 +73,7 @@ class TranslatorService(cotyledon.Service): def __init__(self, worker_id, conf, **kwargs): """Initializer""" - LOG.debug("%s" % self.__class__.__name__) + LOG.debug("{}".format(self.__class__.__name__)) super(TranslatorService, self).__init__(worker_id) self._init(conf, **kwargs) self.running = True @@ -73,6 +82,7 @@ class TranslatorService(cotyledon.Service): self.conf = conf self.Plan = kwargs.get('plan_class') self.kwargs = kwargs + self.opt_schema = load_config_file(str(self.conf.controller.opt_schema_file)) # Set up the RPC service(s) we want to talk to. self.data_service = self.setup_rpc(conf, "data") @@ -138,8 +148,16 @@ class TranslatorService(cotyledon.Service): try: LOG.info(_LI("Requesting plan {} translation").format( plan.id)) - trns = translator.Translator( - self.conf, plan.name, plan.id, plan.template) + template_version = plan.template.get("homing_template_version") + if template_version in VERSIONS['BASE']: + trns = Translator(self.conf, plan.name, plan.id, plan.template) + elif template_version in VERSIONS['GENERIC']: + trns = GenericObjectiveTranslator(self.conf, plan.name, plan.id, plan.template, self.opt_schema) + else: + raise TranslatorException( + "conductor_template_version must be one " + "of: {}".format(', '.join([x for v in VERSIONS.values() for x in v]))) + trns.translate() if trns.ok: @@ -164,7 +182,8 @@ class TranslatorService(cotyledon.Service): plan.status = self.Plan.ERROR _is_success = 'FAILURE' - while 'FAILURE' in _is_success and (self.current_time_seconds() - self.millisec_to_sec(plan.updated)) <= self.conf.messaging_server.timeout: + while 'FAILURE' in _is_success and (self.current_time_seconds() - self.millisec_to_sec(plan.updated)) \ + <= self.conf.messaging_server.timeout: _is_success = plan.update(condition=self.translation_owner_condition) LOG.info(_LI("Changing the template status from translating to {}, " "atomic update response from MUSIC {}").format(plan.status, _is_success)) @@ -214,14 +233,13 @@ class TranslatorService(cotyledon.Service): break # TODO(larry): sychronized clock among Conducotr VMs, or use an offset - elif plan.status == self.Plan.TRANSLATING and \ - (self.current_time_seconds() - self.millisec_to_sec(plan.updated)) > self.conf.messaging_server.timeout: + elif plan.status == self.Plan.TRANSLATING and (self.current_time_seconds() + - self.millisec_to_sec(plan.updated)) \ + > self.conf.messaging_server.timeout: plan.status = self.Plan.TEMPLATE plan.update(condition=self.translating_status_condition) break - - elif plan.timedout: # TODO(jdandrea): How to tell all involved to stop working? # Not enough to just set status. @@ -229,7 +247,7 @@ class TranslatorService(cotyledon.Service): def run(self): """Run""" - LOG.debug("%s" % self.__class__.__name__) + LOG.debug("{}".format(self.__class__.__name__)) # Look for templates to translate from within a thread executor = futurist.ThreadPoolExecutor() @@ -240,12 +258,12 @@ class TranslatorService(cotyledon.Service): def terminate(self): """Terminate""" - LOG.debug("%s" % self.__class__.__name__) + LOG.debug("{}".format(self.__class__.__name__)) self.running = False self._gracefully_stop() super(TranslatorService, self).terminate() def reload(self): """Reload""" - LOG.debug("%s" % self.__class__.__name__) + LOG.debug("{}".format(self.__class__.__name__)) self._restart() diff --git a/conductor/conductor/controller/translator_utils.py b/conductor/conductor/controller/translator_utils.py new file mode 100644 index 0000000..a7452f2 --- /dev/null +++ b/conductor/conductor/controller/translator_utils.py @@ -0,0 +1,104 @@ +# +# ------------------------------------------------------------------------- +# Copyright (C) 2020 Wipro Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +VERSIONS = {'BASE': ["2016-11-01", "2017-10-10", "2018-02-01"], + 'GENERIC': ["2020-08-13"]} +LOCATION_KEYS = ['latitude', 'longitude', 'host_name', 'clli_code'] +INVENTORY_PROVIDERS = ['aai', 'generator'] +INVENTORY_TYPES = ['cloud', 'service', 'transport', 'vfmodule', 'nssi'] +DEFAULT_INVENTORY_PROVIDER = INVENTORY_PROVIDERS[0] +CANDIDATE_KEYS = ['candidate_id', 'cost', 'inventory_type', 'location_id', + 'location_type'] +DEMAND_KEYS = ['filtering_attributes', 'passthrough_attributes', 'default_attributes', 'candidates', 'complex', + 'conflict_identifier', 'customer_id', 'default_cost', 'excluded_candidates', + 'existing_placement', 'flavor', 'inventory_provider', + 'inventory_type', 'port_key', 'region', 'required_candidates', + 'service_id', 'service_resource_id', 'service_subscription', + 'service_type', 'subdivision', 'unique', 'vlan_key'] +CONSTRAINT_KEYS = ['type', 'demands', 'properties'] +CONSTRAINTS = { + # constraint_type: { + # split: split into individual constraints, one per demand + # required: list of required property names, + # optional: list of optional property names, + # thresholds: dict of property/base-unit pairs for threshold parsing + # allowed: dict of keys and allowed values (if controlled vocab); + # only use this for Conductor-controlled values! + # } + 'attribute': { + 'split': True, + 'required': ['evaluate'], + }, + 'threshold': { + 'split': True, + 'required': ['evaluate'], + }, + 'distance_between_demands': { + 'required': ['distance'], + 'thresholds': { + 'distance': 'distance' + }, + }, + 'distance_to_location': { + 'split': True, + 'required': ['distance', 'location'], + 'thresholds': { + 'distance': 'distance' + }, + }, + 'instance_fit': { + 'split': True, + 'required': ['controller'], + 'optional': ['request'], + }, + 'inventory_group': {}, + 'region_fit': { + 'split': True, + 'required': ['controller'], + 'optional': ['request'], + }, + 'zone': { + 'required': ['qualifier', 'category'], + 'optional': ['location'], + 'allowed': {'qualifier': ['same', 'different'], + 'category': ['disaster', 'region', 'complex', 'country', + 'time', 'maintenance']}, + }, + 'vim_fit': { + 'split': True, + 'required': ['controller'], + 'optional': ['request'], + }, + 'hpa': { + 'split': True, + 'required': ['evaluate'], + }, +} +HPA_FEATURES = ['architecture', 'hpa-feature', 'hpa-feature-attributes', + 'hpa-version', 'mandatory', 'directives'] +HPA_OPTIONAL = ['score'] +HPA_ATTRIBUTES = ['hpa-attribute-key', 'hpa-attribute-value', 'operator'] +HPA_ATTRIBUTES_OPTIONAL = ['unit'] +OPTIMIZATION_FUNCTIONS = {'distance_between': ['demand', 'location'], + 'latency_between': ['demand', 'location'], + 'attribute': ['demand', 'attribute']} + + +class TranslatorException(Exception): + pass diff --git a/conductor/conductor/solver/optimizer/best_first.py b/conductor/conductor/solver/optimizer/best_first.py index 2ac6387..7cc64c7 100755 --- a/conductor/conductor/solver/optimizer/best_first.py +++ b/conductor/conductor/solver/optimizer/best_first.py @@ -1,6 +1,7 @@ # # ------------------------------------------------------------------------- # Copyright (c) 2015-2017 AT&T Intellectual Property +# Copyright (C) 2020 Wipro Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -88,7 +89,7 @@ class BestFirst(search.Search): # check closeness for this decision np.set_decision_id(p, candidate.name) - if np.decision_id in list(close_paths.keys()): # Python 3 Conversion -- dict object to list object + if np.decision_id in list(close_paths.keys()): # Python 3 Conversion -- dict object to list object valid_candidate = False ''' for base comparison heuristic ''' @@ -96,6 +97,9 @@ class BestFirst(search.Search): if _objective.goal == "min": if np.total_value >= heuristic_solution.total_value: valid_candidate = False + elif _objective.goal == "max": + if np.total_value <= heuristic_solution.total_value: + valid_candidate = False if valid_candidate is True: open_list.append(np) @@ -141,7 +145,7 @@ class BestFirst(search.Search): best_resource = None for candidate in candidate_list: _decision_path.decisions[demand.name] = candidate - _objective.compute(_decision_path) #TODO call the compute of latencyBetween + _objective.compute(_decision_path) if _objective.goal == "min": if _decision_path.total_value < bound_value: bound_value = _decision_path.total_value diff --git a/conductor/conductor/solver/optimizer/fit_first.py b/conductor/conductor/solver/optimizer/fit_first.py index 62e011d..b558ce6 100755 --- a/conductor/conductor/solver/optimizer/fit_first.py +++ b/conductor/conductor/solver/optimizer/fit_first.py @@ -1,6 +1,7 @@ # # ------------------------------------------------------------------------- # Copyright (c) 2015-2017 AT&T Intellectual Property +# Copyright (C) 2020 Wipro Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,7 +24,6 @@ import time from conductor.solver.optimizer import decision_path as dpath from conductor.solver.optimizer import search -from conductor.solver.triage_tool.triage_data import TriageData LOG = log.getLogger(__name__) @@ -98,9 +98,8 @@ class FitFirst(search.Search): candidate_version = candidate \ .get("cloud_region_version").encode('utf-8') if _decision_path.total_value < bound_value or \ - (_decision_path.total_value == bound_value and - self._compare_version(candidate_version, - version_value) > 0): + (_decision_path.total_value == bound_value and self._compare_version(candidate_version, + version_value) > 0): bound_value = _decision_path.total_value version_value = candidate_version best_resource = candidate @@ -116,6 +115,11 @@ class FitFirst(search.Search): bound_value = _decision_path.total_value best_resource = candidate + elif _objective.goal == "max": + if _decision_path.total_value > bound_value: + bound_value = _decision_path.total_value + best_resource = candidate + # Rollback if we don't have any candidate picked for # the demand. if best_resource is None: @@ -124,7 +128,7 @@ class FitFirst(search.Search): # candidate) back in the list so that it can be picked # up in the next iteration of the recursion _demand_list.insert(0, demand) - self.triageSolver.rollBackStatus(_decision_path.current_demand,_decision_path) + self.triageSolver.rollBackStatus(_decision_path.current_demand, _decision_path) return None # return None back to the recursion else: # best resource is found, add to the decision path diff --git a/conductor/conductor/solver/optimizer/greedy.py b/conductor/conductor/solver/optimizer/greedy.py index eae1b12..5c82164 100755 --- a/conductor/conductor/solver/optimizer/greedy.py +++ b/conductor/conductor/solver/optimizer/greedy.py @@ -2,6 +2,7 @@ # # ------------------------------------------------------------------------- # Copyright (c) 2015-2017 AT&T Intellectual Property +# Copyright (C) 2020 Wipro Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -55,6 +56,10 @@ class Greedy(search.Search): if decision_path.total_value < bound_value: bound_value = decision_path.total_value best_resource = candidate + elif _objective.goal == "max": + if decision_path.total_value > bound_value: + bound_value = decision_path.total_value + best_resource = candidate if best_resource is not None: decision_path.decisions[demand.name] = best_resource diff --git a/conductor/conductor/solver/request/functions/__init__.py b/conductor/conductor/solver/request/functions/__init__.py index e69de29..b3be6b9 100755 --- a/conductor/conductor/solver/request/functions/__init__.py +++ b/conductor/conductor/solver/request/functions/__init__.py @@ -0,0 +1,26 @@ +# +# ------------------------------------------------------------------------- +# Copyright (C) 2020 Wipro Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +from conductor.solver.request.functions import aic_version +from conductor.solver.request.functions import attribute +from conductor.solver.request.functions import cloud_version +from conductor.solver.request.functions import cost +from conductor.solver.request.functions import distance_between +from conductor.solver.request.functions import hpa_score +from conductor.solver.request.functions import latency_between diff --git a/conductor/conductor/solver/request/functions/attribute.py b/conductor/conductor/solver/request/functions/attribute.py new file mode 100644 index 0000000..4307572 --- /dev/null +++ b/conductor/conductor/solver/request/functions/attribute.py @@ -0,0 +1,32 @@ +# +# ------------------------------------------------------------------------- +# Copyright (C) 2020 Wipro Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + + +class Attribute(object): + + def __init__(self, _type): + self.func_type = _type + + def compute(self, candidate, attribute): + return candidate.get(attribute) + + def get_args_from_params(self, decision_path, request, params): + demand = params.get('demand') + attribute = params.get('attribute') + return decision_path.decisions[demand], attribute diff --git a/conductor/conductor/solver/request/functions/distance_between.py b/conductor/conductor/solver/request/functions/distance_between.py index 66413be..cabe640 100755 --- a/conductor/conductor/solver/request/functions/distance_between.py +++ b/conductor/conductor/solver/request/functions/distance_between.py @@ -2,6 +2,7 @@ # # ------------------------------------------------------------------------- # Copyright (c) 2015-2017 AT&T Intellectual Property +# Copyright (C) 2020 Wipro Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,12 +19,14 @@ # ------------------------------------------------------------------------- # +from conductor.solver.request.functions.location_function import LocationFunction from conductor.solver.utils import utils -class DistanceBetween(object): +class DistanceBetween(LocationFunction): def __init__(self, _type): + super(DistanceBetween, self).__init__() self.func_type = _type self.loc_a = None diff --git a/conductor/conductor/solver/request/functions/latency_between.py b/conductor/conductor/solver/request/functions/latency_between.py index 15f5489..1f9d368 100644 --- a/conductor/conductor/solver/request/functions/latency_between.py +++ b/conductor/conductor/solver/request/functions/latency_between.py @@ -1,6 +1,7 @@ # # ------------------------------------------------------------------------- # Copyright (c) 2015-2018 AT&T Intellectual Property +# Copyright (C) 2020 Wipro Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,10 +18,13 @@ # ------------------------------------------------------------------------- # +from conductor.solver.request.functions.location_function import LocationFunction from conductor.solver.utils import utils -class LatencyBetween(object): + +class LatencyBetween(LocationFunction): def __init__(self, _type): + super(LatencyBetween, self).__init__() self.func_type = _type self.loc_a = None @@ -31,5 +35,3 @@ class LatencyBetween(object): latency = utils.compute_latency_score(_loc_a, _loc_z, self.region_group) return latency - - diff --git a/conductor/conductor/solver/request/functions/location_function.py b/conductor/conductor/solver/request/functions/location_function.py new file mode 100644 index 0000000..c719602 --- /dev/null +++ b/conductor/conductor/solver/request/functions/location_function.py @@ -0,0 +1,35 @@ +# +# ------------------------------------------------------------------------- +# Copyright (C) 2020 Wipro Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + + +class LocationFunction(object): + """Super class for functions that applies on locations.""" + + def __init__(self): + pass + + def get_args_from_params(self, decision_path, request, params): + demand = params.get('demand') + location = params.get('location') + + resource = decision_path.decisions[demand] + loc_a = request.cei.get_candidate_location(resource) + loc_z = request.location.get(location) + + return loc_a, loc_z diff --git a/conductor/conductor/solver/request/generic_objective.py b/conductor/conductor/solver/request/generic_objective.py new file mode 100644 index 0000000..1c2d922 --- /dev/null +++ b/conductor/conductor/solver/request/generic_objective.py @@ -0,0 +1,78 @@ +# +# ------------------------------------------------------------------------- +# Copyright (C) 2020 Wipro Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +from conductor.solver.request import functions +from conductor.solver.utils.utils import OPERATOR_FUNCTIONS + +GOALS = {'minimize': 'min', + 'maximize': 'max'} + + +def get_method_class(function_name): + module_name = getattr(functions, function_name) + return getattr(module_name, dir(module_name)[0]) + + +def get_normalized_value(value, start, end): + return (value - start) / (end - start) + + +class GenericObjective(object): + + def __init__(self, objective_function): + self.goal = GOALS[objective_function.get('goal')] + self.operation_function = objective_function.get('operation_function') + self.operand_list = [] # keeping this for compatibility with the solver + + def compute(self, _decision_path, _request): + value = self.compute_operation_function(self.operation_function, _decision_path, _request) + _decision_path.cumulated_value = value + _decision_path.total_value = \ + _decision_path.cumulated_value + \ + _decision_path.heuristic_to_go_value + + def compute_operation_function(self, operation_function, _decision_path, _request): + operator = operation_function.get('operator') + operands = operation_function.get('operands') + + result_list = [] + + for operand in operands: + if 'operation_function' in operand: + value = self.compute_operation_function(operand.get('operation_function'), + _decision_path, _request) + else: + function_name = operand.get('function') + function_class = get_method_class(function_name) + function = function_class(function_name) + args = function.get_args_from_params(_decision_path, _request, + operand.get('params')) + value = function.compute(*args) + + if 'normalization' in operand: + normalization = operand.get('normalization') + value = get_normalized_value(value, normalization.get('start'), + normalization.get('end')) + + if 'weight' in operand: + value = value * operand.get("weight") + + result_list.append(value) + + return OPERATOR_FUNCTIONS.get(operator)(result_list) diff --git a/conductor/conductor/solver/request/parser.py b/conductor/conductor/solver/request/parser.py index 13fd5e6..6bf2028 100755 --- a/conductor/conductor/solver/request/parser.py +++ b/conductor/conductor/solver/request/parser.py @@ -19,11 +19,10 @@ # ------------------------------------------------------------------------- # - -# import json import collections import operator -import random + +from oslo_log import log from conductor.solver.optimizer.constraints \ import access_distance as access_dist @@ -36,28 +35,30 @@ from conductor.solver.optimizer.constraints \ import inventory_group from conductor.solver.optimizer.constraints \ import service as service_constraint +from conductor.solver.optimizer.constraints import threshold from conductor.solver.optimizer.constraints import vim_fit from conductor.solver.optimizer.constraints import zone -from conductor.solver.optimizer.constraints import threshold from conductor.solver.request import demand -from conductor.solver.request import objective from conductor.solver.request.functions import aic_version from conductor.solver.request.functions import cost from conductor.solver.request.functions import distance_between from conductor.solver.request.functions import hpa_score from conductor.solver.request.functions import latency_between +from conductor.solver.request import generic_objective from conductor.solver.request import objective from conductor.solver.triage_tool.traige_latency import TriageLatency -from oslo_log import log + LOG = log.getLogger(__name__) +V2_IDS = ["2020-08-13"] + # FIXME(snarayanan): This is really a SolverRequest (or Request) object class Parser(object): - demands = None # type: Dict[Any, Any] - locations = None # type: Dict[Any, Any] + demands = None + locations = None obj_func_param = None def __init__(self, _region_gen=None): @@ -241,6 +242,9 @@ class Parser(object): if "objective" not in json_template["conductor_solver"] \ or not json_template["conductor_solver"]["objective"]: self.objective = objective.Objective() + elif json_template["conductor_solver"]["version"] in V2_IDS: + objective_function = json_template["conductor_solver"]["objective"] + self.objective = generic_objective.GenericObjective(objective_function) else: input_objective = json_template["conductor_solver"]["objective"] self.objective = objective.Objective() @@ -305,11 +309,11 @@ class Parser(object): self.latencyTriage.updateTriageLatencyDB(self.plan_id, self.request_id) def assign_region_group_weight(self, countries, regions): - """ assign the latency group value to the country and returns a map""" + """assign the latency group value to the country and returns a map""" LOG.info("Processing Assigning Latency Weight to Countries ") - countries = self.resolve_countries(countries, regions, - self.get_candidate_country_list()) # resolve the countries based on region type + # resolve the countries based on region type + countries = self.resolve_countries(countries, regions, self.get_candidate_country_list()) region_latency_weight = collections.OrderedDict() weight = 0 @@ -320,7 +324,8 @@ class Parser(object): try: l_weight = '' for i, e in enumerate(countries): - if e is None: continue + if e is None: + continue for k, x in enumerate(e.split(',')): region_latency_weight[x] = weight l_weight += x + " : " + str(weight) @@ -406,7 +411,6 @@ class Parser(object): def drop_no_latency_rule_candidates(self, diff_bw_candidates_and_countries): - cadidate_list_ = list() temp_candidates = dict() for demand_id, demands in self.demands.items(): @@ -423,30 +427,11 @@ class Parser(object): if demand_id in self.obj_func_param and candidate["country"] in diff_bw_candidates_and_countries: droped_candidates += candidate['candidate_id'] droped_candidates += ',' - self.latencyTriage.latencyDroppedCandiate(candidate['candidate_id'], demand_id, reason="diff_bw_candidates_and_countries,Latecy weight ") + self.latencyTriage.latencyDroppedCandiate(candidate['candidate_id'], demand_id, + reason="diff_bw_candidates_and_countries,Latecy weight ") self.demands[demand_id].resources.pop(candidate['candidate_id']) LOG.info("dropped " + droped_candidates) - # for demand_id, candidate_list in self.demands: - # LOG.info("Candidates for demand " + demand_id) - # cadidate_list_ = self.demands[demand_id]['candidates'] - # droped_candidates = '' - # xlen = cadidate_list_.__len__() - 1 - # len = xlen - # # LOG.info("Candidate List Length "+str(len)) - # for i in range(len + 1): - # # LOG.info("iteration " + i) - # LOG.info("Candidate Country " + cadidate_list_[xlen]["country"]) - # if cadidate_list_[xlen]["country"] in diff_bw_candidates_and_countries: - # droped_candidates += cadidate_list_[xlen]["country"] - # droped_candidates += ',' - # self.demands[demand_id]['candidates'].remove(cadidate_list_[xlen]) - # # filter(lambda candidate: candidate in candidate_list["candidates"]) - # # LOG.info("Droping Cadidate not eligible for latency weight. Candidate ID " + cadidate_list_[xlen]["candidate_id"] + " Candidate Country: "+cadidate_list_[xlen]["country"]) - # xlen = xlen - 1 - # if xlen < 0: break - # LOG.info("Dropped Candidate Countries " + droped_candidates + " from demand " + demand_id) - def process_wildcard_rules(self, candidates_country_list, countries_list, ): LOG.info("Processing the rules for " + countries_list.__getitem__(countries_list.__len__() - 1)) candidate_countries = '' @@ -482,9 +467,10 @@ class Parser(object): LOG.info("Available countries after processing diff between " + ac) def filter_invalid_rules(self, countries_list, regions_map): - invalid_rules = list(); + invalid_rules = list() for i, e in enumerate(countries_list): - if e is None: continue + if e is None: + continue for k, region in enumerate(e.split(',')): LOG.info("Processing the Rule for " + region) diff --git a/conductor/conductor/solver/utils/utils.py b/conductor/conductor/solver/utils/utils.py index c995eec..cedf0a7 100755 --- a/conductor/conductor/solver/utils/utils.py +++ b/conductor/conductor/solver/utils/utils.py @@ -1,6 +1,7 @@ # # ------------------------------------------------------------------------- # Copyright (c) 2015-2017 AT&T Intellectual Property +# Copyright (C) 2020 Wipro Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,7 +18,9 @@ # ------------------------------------------------------------------------- # +from functools import reduce import math +import operator from oslo_log import log @@ -32,6 +35,11 @@ OPERATIONS = {'gte': lambda x, y: x >= y, } +OPERATOR_FUNCTIONS = {'sum': lambda x: reduce(operator.add, x), + 'min': lambda x: reduce(lambda a, b: a if a < b else b, x), + 'max': lambda x: reduce(lambda a, b: a if a < b else b, x)} + + def compute_air_distance(_src, _dst): """Compute Air Distance @@ -40,14 +48,12 @@ def compute_air_distance(_src, _dst): output: air distance as km """ distance = 0.0 - latency_score = 0.0 if _src == _dst: return distance radius = 6371.0 # km - dlat = math.radians(_dst[0] - _src[0]) dlon = math.radians(_dst[1] - _src[1]) a = math.sin(dlat / 2.0) * math.sin(dlat / 2.0) + \ @@ -60,18 +66,18 @@ def compute_air_distance(_src, _dst): return distance -def compute_latency_score(_src,_dst, _region_group): +def compute_latency_score(_src, _dst, _region_group): """Compute the Network latency score between src and dst""" earth_half_circumference = 20000 region_group_weight = _region_group.get(_dst[2]) - if region_group_weight == 0 or region_group_weight is None : + if region_group_weight == 0 or region_group_weight is None: LOG.debug("Computing the latency score based on distance between : ") - latency_score = compute_air_distance(_src,_dst) - elif _region_group > 0 : + latency_score = compute_air_distance(_src, _dst) + elif _region_group > 0: LOG.debug("Computing the latency score ") latency_score = compute_air_distance(_src, _dst) + region_group_weight * earth_half_circumference - LOG.debug("Finished Computing the latency score: "+str(latency_score)) + LOG.debug("Finished Computing the latency score: " + str(latency_score)) return latency_score diff --git a/conductor/conductor/tests/unit/controller/opt_schema.json b/conductor/conductor/tests/unit/controller/opt_schema.json new file mode 100644 index 0000000..cdf04de --- /dev/null +++ b/conductor/conductor/tests/unit/controller/opt_schema.json @@ -0,0 +1,138 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "The root schema comprises the entire JSON document.", + "required": [ + "goal", + "operation_function" + ], + "title": "The root schema", + "properties": { + "goal": { + "description": "Goal of the optimization.", + "enum": [ + "minimize", + "maximize" + ], + "title": "The goal schema", + "type": "string" + }, + "operation_function": { + "$id": "#operation_function", + "description": "The operation function that has to be optimized.", + "required": [ + "operator", + "operands" + ], + "title": "The operation_function schema", + "properties": { + "operator": { + "description": "The operation which will be a part of the objective function.", + "enum": [ + "sum", + "min", + "max" + ], + "title": "The operator schema", + "type": "string" + }, + "operands": { + "description": "The operand on which the operation is to be performed.", + "title": "The operands schema", + "type": "array", + "additionalItems": true, + "items": { + "anyOf": [ + { + "default": {}, + "description": "An explanation about the purpose of this instance.", + "required": [ + "function", + "params" + ], + "title": "function operand schema", + "properties": { + "function": { + "default": "", + "description": "Function to be performed on the parameters", + "enum": [ + "distance_between", + "latency_between", + "attribute" + ], + "title": "The function schema", + "type": "string" + }, + "weight": { + "default": 1.0, + "description": "Weight for the operand.", + "title": "The weight schema", + "type": "number" + }, + "params": { + "description": "key-value pair which will be passed as kwargs to the function.", + "title": "The params schema", + "type": "object", + "additionalProperties": true + }, + "normalization": { + "description": "Set of values used to normalize the operand.", + "$id": "#normalization", + "required": [ + "start", + "end" + ], + "title": "The normalization schema", + "properties": { + "start": { + "description": "Start of the range.", + "title": "The start schema", + "type": "number" + }, + "end": { + "description": "End of the range.", + "title": "The end schema", + "type": "number" + } + }, + "additionalProperties": true + } + }, + "additionalProperties": true + }, + { + "description": "operation function operand.", + "required": [ + "operation_function" + ], + "title": "The operation function operand schema", + "properties": { + "operation_function": { + "description": "The operation function which same as the top level object.", + "title": "The operation_function schema", + "$ref": "#/properties/operation_function", + "additionalProperties": true + }, + "normalization": { + "description": "Set of values used to normalize the operand.", + "title": "The normalization schema", + "$ref": "#/properties/operation_function/properties/operands/items/anyOf/0/properties/normalization", + "additionalProperties": true + }, + "weight": { + "default": 1.0, + "description": "An explanation about the purpose of this instance.", + "title": "The weight schema", + "type": "number" + } + }, + "additionalProperties": true + } + ] + } + } + }, + "additionalProperties": true + } + }, + "additionalProperties": true +} diff --git a/conductor/conductor/tests/unit/controller/template_v2.json b/conductor/conductor/tests/unit/controller/template_v2.json new file mode 100644 index 0000000..0884928 --- /dev/null +++ b/conductor/conductor/tests/unit/controller/template_v2.json @@ -0,0 +1,150 @@ +{ + "constraints": { + "cloud_version_capabilities": { + "demands": [ + "vGMuxInfra" + ], + "properties": { + "evaluate": { + "cloud_provider": "AWS", + "cloud_version": "1.11.84" + } + }, + "type": "attribute" + }, + "colocation": { + "demands": [ + "vGMuxInfra", + "vG" + ], + "properties": { + "category": "region", + "qualifier": "same" + }, + "type": "zone" + }, + "constraint_vgmux_customer": { + "demands": [ + "vGMuxInfra" + ], + "properties": { + "distance": "<\u00a0100\u00a0km", + "location": "customer_loc" + }, + "type": "distance_to_location" + }, + "numa_cpu_pin_capabilities": { + "demands": [ + "vG" + ], + "properties": { + "evaluate": { + "numa_topology": "numa_spanning", + "vcpu_pinning": true + } + }, + "type": "attribute" + } + }, + "demands": { + "vG": [ + { + "attributes": { + "customer_id": "some_company", + "equipment_type": "vG", + "modelId": "vG_model_id" + }, + "excluded_candidates": [ + { + "candidate_id": "1ac71fb8-ad43-4e16-9459-c3f372b8236d" + } + ], + "existing_placement": [ + { + "candidate_id": "21d5f3e8-e714-4383-8f99-cc480144505a" + } + ], + "inventory_provider": "aai", + "inventory_type": "service" + }, + { + "inventory_provider": "aai", + "inventory_type": "cloud" + } + ], + "vGMuxInfra": [ + { + "attributes": { + "customer_id": "some_company", + "equipment_type": "vG_Mux" + }, + "excluded_candidates": [ + { + "candidate_id": "1ac71fb8-ad43-4e16-9459-c3f372b8236d" + } + ], + "existing_placement": [ + { + "candidate_id": "21d5f3e8-e714-4383-8f99-cc480144505a" + } + ], + "inventory_provider": "aai", + "inventory_type": "service" + } + ] + }, + "homing_template_version": "2020-08-13", + "locations": { + "customer_loc": { + "latitude": { + "get_param": "customer_lat" + }, + "longitude": { + "get_param": "customer_long" + } + } + }, + "optimization": { + "goal": "minimize", + "operation_function": { + "operands": [ + { + "function": "distance_between", + "params": { + "demand": "vG", + "location": "customer_loc" + }, + "weight": 1.0 + }, + { + "normalization": { + "end": 5, + "start": 50 + }, + "operation_function": { + "operands": [ + { + "function": "attribute", + "params": { + "attribute": "latency", + "demand": "vG" + }, + "weight": 1.0 + } + ], + "operator": "sum" + }, + "weight": 1.0 + } + ], + "operator": "sum" + } + }, + "parameters": { + "customer_lat": 32.89748, + "customer_long": -97.040443, + "service_id": "vcpe_service_id", + "service_name": "Residential vCPE" + } +} + diff --git a/conductor/conductor/tests/unit/controller/test_generic_objective_translator.py b/conductor/conductor/tests/unit/controller/test_generic_objective_translator.py new file mode 100644 index 0000000..f9ed79b --- /dev/null +++ b/conductor/conductor/tests/unit/controller/test_generic_objective_translator.py @@ -0,0 +1,101 @@ +# +# ------------------------------------------------------------------------- +# Copyright (C) 2020 Wipro Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# +"""Test classes for translator V2""" + +import copy +import json +from mock import patch +import os +import unittest +import uuid +from oslo_config import cfg + +from conductor.controller.translator_utils import TranslatorException +from conductor.controller.generic_objective_translator import GenericObjectiveTranslator + +DIR = os.path.dirname(__file__) + + +class TestGenericObjectiveTranslator(unittest.TestCase): + + def setUp(self): + + with open(os.path.join(DIR, 'template_v2.json'), 'r') as tpl: + self.template = json.loads(tpl.read()) + with open(os.path.join(DIR, 'opt_schema.json'), 'r') as sch: + self.opt_schema = json.loads(sch.read()) + + def tearDown(self): + pass + + @patch('conductor.common.music.model.base.Base.table_create') + @patch('conductor.controller.translator.Translator.parse_demands') + def test_translator_template(self, mock_table, mock_parse): + cfg.CONF.set_override('keyspace', 'conductor') + cfg.CONF.set_override('keyspace', 'conductor_rpc', 'messaging_server') + cfg.CONF.set_override('concurrent', True, 'controller') + cfg.CONF.set_override('certificate_authority_bundle_file', '../AAF_RootCA.cer', 'music_api') + conf = cfg.CONF + translator = GenericObjectiveTranslator(conf, 'v2_test', str(uuid.uuid4()), self.template, + self.opt_schema) + translator.translate() + self.assertEqual(translator._ok, True) + self.assertEqual(self.template.get('optimization'), + translator._translation.get('conductor_solver').get('objective')) + + @patch('conductor.common.music.model.base.Base.table_create') + def test_translator_error_version(self, mock_table): + temp = copy.deepcopy(self.template) + temp["homing_template_version"] = "2020-04-04" + conf = cfg.CONF + translator = GenericObjectiveTranslator(conf, 'v2_test', str(uuid.uuid4()), temp, + self.opt_schema) + translator.create_components() + self.assertRaises(TranslatorException, translator.validate_components) + + @patch('conductor.common.music.model.base.Base.table_create') + def test_translator_no_optimization(self, mock_table): + conf = cfg.CONF + translator = GenericObjectiveTranslator(conf, 'v2_test', str(uuid.uuid4()), self.template, + self.opt_schema) + self.assertEqual(None, translator.parse_optimization({})) + + @patch('conductor.common.music.model.base.Base.table_create') + def test_translator_wrong_opt(self, mock_table): + opt = {"goal": "nothing"} + conf = cfg.CONF + translator = GenericObjectiveTranslator(conf, 'v2_test', str(uuid.uuid4()), self.template, + self.opt_schema) + self.assertRaises(TranslatorException, translator.parse_optimization, optimization=opt) + + @patch('conductor.common.music.model.base.Base.table_create') + @patch('conductor.controller.translator.Translator.parse_demands') + def test_translator_incorrect_demand(self, mock_table, mock_parse): + templ = copy.deepcopy(self.template) + templ["optimization"]["operation_function"]["operands"][0]["params"]["demand"] = "vF" + conf = cfg.CONF + translator = GenericObjectiveTranslator(conf, 'v2_test', str(uuid.uuid4()), templ, + self.opt_schema) + translator.create_components() + self.assertRaises(TranslatorException, translator.parse_optimization, + optimization=templ["optimization"]) + + +if __name__ == '__main__': + unittest.main() diff --git a/conductor/conductor/tests/unit/controller/test_translator_svc.py b/conductor/conductor/tests/unit/controller/test_translator_svc.py index a315c4b..f256991 100644 --- a/conductor/conductor/tests/unit/controller/test_translator_svc.py +++ b/conductor/conductor/tests/unit/controller/test_translator_svc.py @@ -18,6 +18,7 @@ # """Test classes for translator_svc""" +import os import unittest import uuid import time @@ -50,6 +51,9 @@ class TestTranslatorServiceNoException(unittest.TestCase): cfg.CONF.set_override('timeout', 10, 'controller') cfg.CONF.set_override('limit', 1, 'controller') cfg.CONF.set_override('concurrent', True, 'controller') + cfg.CONF.set_override('opt_schema_file', + os.path.join(os.path.dirname(__file__), 'opt_schema.json'), + 'controller') cfg.CONF.set_override('keyspace', 'conductor_rpc', 'messaging_server') cfg.CONF.set_override('certificate_authority_bundle_file', '../AAF_RootCA.cer', 'music_api') @@ -77,7 +81,6 @@ class TestTranslatorServiceNoException(unittest.TestCase): self.translator_svc.translate(self.mock_plan) self.assertEqual(self.mock_plan.status, 'translated') - @patch('conductor.controller.translator.Translator.translate') @patch('conductor.controller.translator.Translator.error_message') @patch('conductor.common.music.model.base.Base.update') diff --git a/conductor/conductor/tests/unit/solver/request/functions/test_attribute.py b/conductor/conductor/tests/unit/solver/request/functions/test_attribute.py new file mode 100644 index 0000000..ccdba78 --- /dev/null +++ b/conductor/conductor/tests/unit/solver/request/functions/test_attribute.py @@ -0,0 +1,47 @@ +# +# ------------------------------------------------------------------------- +# Copyright (C) 2020 Wipro Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import unittest + +from conductor.solver.optimizer.decision_path import DecisionPath +from conductor.solver.request.parser import Parser +from conductor.solver.request.functions.attribute import Attribute + + +class TestAttribute(unittest.TestCase): + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_attribute(self): + candidate = {"canidate_id": "1234", + "candidate_type": "nsi", + "latency": 5, + "reliability": 99.9} + decisions = {"urllc": candidate} + decision_path = DecisionPath() + decision_path.decisions = decisions + params = {"demand": "urllc", + "attribute": "latency"} + attribute = Attribute("attribute") + args = attribute.get_args_from_params(decision_path, Parser(), params) + self.assertEqual(5, attribute.compute(*args)) diff --git a/conductor/conductor/tests/unit/solver/request/objective.json b/conductor/conductor/tests/unit/solver/request/objective.json new file mode 100644 index 0000000..2282a99 --- /dev/null +++ b/conductor/conductor/tests/unit/solver/request/objective.json @@ -0,0 +1,97 @@ +[ + { + "goal": "minimize", + "operation_function": { + "operands": [ + { + "function": "attribute", + "params": { + "attribute": "latency", + "demand": "urllc_core" + } + } + ], + "operator": "sum" + } + }, + { + "goal": "maximize", + "operation_function": { + "operands": [ + { + "normalization": { + "end": 1000, + "start": 100 + }, + "operation_function": { + "operands": [ + { + "function": "attribute", + "params": { + "attribute": "throughput", + "demand": "urllc_core" + }, + "weight": 1.0 + }, + { + "function": "attribute", + "params": { + "attribute": "throughput", + "demand": "urllc_ran" + }, + "weight": 1.0 + }, + { + "function": "attribute", + "params": { + "attribute": "throughput", + "demand": "urllc_transport" + }, + "weight": 1.0 + } + ], + "operator": "min" + }, + "weight": 2.0 + }, + { + "normalization": { + "end": 5, + "start": 50 + }, + "operation_function": { + "operands": [ + { + "function": "attribute", + "params": { + "attribute": "latency", + "demand": "urllc_core" + }, + "weight": 1.0 + }, + { + "function": "attribute", + "params": { + "attribute": "latency", + "demand": "urllc_ran" + }, + "weight": 1.0 + }, + { + "function": "attribute", + "params": { + "attribute": "latency", + "demand": "urllc_transport" + }, + "weight": 1.0 + } + ], + "operator": "sum" + }, + "weight": 1.0 + } + ], + "operator": "sum" + } + } +] \ No newline at end of file diff --git a/conductor/conductor/tests/unit/solver/request/test_generic_objective.py b/conductor/conductor/tests/unit/solver/request/test_generic_objective.py new file mode 100644 index 0000000..8e4597e --- /dev/null +++ b/conductor/conductor/tests/unit/solver/request/test_generic_objective.py @@ -0,0 +1,71 @@ +# +# ------------------------------------------------------------------------- +# Copyright (C) 2020 Wipro Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import json +import os +import unittest + +from conductor.solver.optimizer.decision_path import DecisionPath +from conductor.solver.request.generic_objective import GenericObjective +from conductor.solver.request.parser import Parser + +BASE_DIR = os.path.dirname(__file__) + + +class TestGenericObjective(unittest.TestCase): + + def setUp(self): + with open(os.path.join(BASE_DIR, 'objective.json'), 'r') as obj: + self.objective_functions = json.loads(obj.read()) + + def tearDown(self): + pass + + def test_objective(self): + + expected = [10, 0.6] + candidate_core = {"candidate_id": "12345", + "candidate_type": "nssi", + "latency": 10, + "throughput": 200} + candidate_ran = {"candidate_id": "12345", + "candidate_type": "nssi", + "latency": 15, + "throughput": 300} + candidate_transport = {"candidate_id": "12345", + "candidate_type": "nssi", + "latency": 8, + "throughput": 400} + + decisions = {"urllc_core": candidate_core, + "urllc_ran": candidate_ran, + "urllc_transport": candidate_transport} + + decision_path = DecisionPath() + decision_path.decisions = decisions + request = Parser() + + actual = [] + for objective_function in self.objective_functions: + objective = GenericObjective(objective_function) + objective.compute(decision_path, request) + actual.append(decision_path.cumulated_value) + + self.assertEqual(expected, actual) + diff --git a/conductor/etc/conductor/opt_schema.json b/conductor/etc/conductor/opt_schema.json new file mode 100644 index 0000000..cdf04de --- /dev/null +++ b/conductor/etc/conductor/opt_schema.json @@ -0,0 +1,138 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "The root schema comprises the entire JSON document.", + "required": [ + "goal", + "operation_function" + ], + "title": "The root schema", + "properties": { + "goal": { + "description": "Goal of the optimization.", + "enum": [ + "minimize", + "maximize" + ], + "title": "The goal schema", + "type": "string" + }, + "operation_function": { + "$id": "#operation_function", + "description": "The operation function that has to be optimized.", + "required": [ + "operator", + "operands" + ], + "title": "The operation_function schema", + "properties": { + "operator": { + "description": "The operation which will be a part of the objective function.", + "enum": [ + "sum", + "min", + "max" + ], + "title": "The operator schema", + "type": "string" + }, + "operands": { + "description": "The operand on which the operation is to be performed.", + "title": "The operands schema", + "type": "array", + "additionalItems": true, + "items": { + "anyOf": [ + { + "default": {}, + "description": "An explanation about the purpose of this instance.", + "required": [ + "function", + "params" + ], + "title": "function operand schema", + "properties": { + "function": { + "default": "", + "description": "Function to be performed on the parameters", + "enum": [ + "distance_between", + "latency_between", + "attribute" + ], + "title": "The function schema", + "type": "string" + }, + "weight": { + "default": 1.0, + "description": "Weight for the operand.", + "title": "The weight schema", + "type": "number" + }, + "params": { + "description": "key-value pair which will be passed as kwargs to the function.", + "title": "The params schema", + "type": "object", + "additionalProperties": true + }, + "normalization": { + "description": "Set of values used to normalize the operand.", + "$id": "#normalization", + "required": [ + "start", + "end" + ], + "title": "The normalization schema", + "properties": { + "start": { + "description": "Start of the range.", + "title": "The start schema", + "type": "number" + }, + "end": { + "description": "End of the range.", + "title": "The end schema", + "type": "number" + } + }, + "additionalProperties": true + } + }, + "additionalProperties": true + }, + { + "description": "operation function operand.", + "required": [ + "operation_function" + ], + "title": "The operation function operand schema", + "properties": { + "operation_function": { + "description": "The operation function which same as the top level object.", + "title": "The operation_function schema", + "$ref": "#/properties/operation_function", + "additionalProperties": true + }, + "normalization": { + "description": "Set of values used to normalize the operand.", + "title": "The normalization schema", + "$ref": "#/properties/operation_function/properties/operands/items/anyOf/0/properties/normalization", + "additionalProperties": true + }, + "weight": { + "default": 1.0, + "description": "An explanation about the purpose of this instance.", + "title": "The weight schema", + "type": "number" + } + }, + "additionalProperties": true + } + ] + } + } + }, + "additionalProperties": true + } + }, + "additionalProperties": true +} diff --git a/conductor/requirements.txt b/conductor/requirements.txt index 42a0d89..62630cb 100644 --- a/conductor/requirements.txt +++ b/conductor/requirements.txt @@ -27,3 +27,4 @@ onapsmsclient>=0.0.4 Flask>=0.11.1 prometheus-client>=0.3.1 pycryptodome==3.9.7 +jsonschema>=3.2.0 diff --git a/conductor/tox.ini b/conductor/tox.ini index d6d120d..bd9de98 100644 --- a/conductor/tox.ini +++ b/conductor/tox.ini @@ -62,10 +62,10 @@ commands = bash -x oslo_debug_helper {posargs} [flake8] select = E,H,W,F max-line-length = 119 -exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,install-guide,*/tests/*,conductor/data/service.py +exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,install-guide,*/tests/*,__init__.py,conductor/data/service.py show-source = True -ignore= W503 #conflict with W504 -per-file-ignores= conductor/data/plugins/inventory_provider/aai.py:F821 +ignore = W503 #conflict with W504 +per-file-ignores = conductor/data/plugins/inventory_provider/aai.py:F821 [hacking] import_exceptions = conductor.common.i18n -- 2.16.6