Data Service RPC for HPA

Implemented RPC for get candidate list with hpa flavor mapping
Added unit tests for get_candidates_with_hpa

Issue-ID: OPTFRA-183
Change-Id: I9db74499ba964341368542b339163c3803e0924e
Signed-off-by: Dileep Ranganathan <dileep.ranganathan@intel.com>

diff --git a/conductor/conductor/data/service.py b/conductor/conductor/data/service.py
index acb4233..9617217 100644 (file)
--- a/conductor/conductor/data/service.py
+++ b/conductor/conductor/data/service.py
@@ -21,17 +21,17 @@
 # import os
 
 import cotyledon
-from oslo_config import cfg
-from oslo_log import log
-# from stevedore import driver
-
+from conductor import messaging
 # from conductor import __file__ as conductor_root
 from conductor.common.music import messaging as music_messaging
+from conductor.common.utils import conductor_logging_util as log_util
 from conductor.data.plugins.inventory_provider import extensions as ip_ext
 from conductor.data.plugins.service_controller import extensions as sc_ext
 from conductor.i18n import _LE, _LI, _LW
-from conductor import messaging
-from conductor.common.utils import conductor_logging_util as log_util
+from oslo_config import cfg
+from oslo_log import log
+
+# from stevedore import driver
 # from conductor.solver.resource import region
 # from conductor.solver.resource import service
 
@@ -427,6 +427,70 @@ class DataEndpoint(object):
                 candidate_list, self.ip_ext_manager.names()[0]))
         return {'response': candidate_list, 'error': False}
 
+    def get_candidates_with_hpa(self, ctx, arg):
+        '''
+        RPC for getting candidates flavor mapping for matching hpa
+        :param ctx: context
+        :param arg: contains input passed from client side for RPC call
+        :return: response candidate_list with matching label to flavor mapping
+        '''
+        error = False
+        candidate_list = arg["candidate_list"]
+        label_name = arg["label_name"]
+        features = arg["features"]
+        discard_set = set()
+        for candidate in candidate_list:
+            # perform this check only for cloud candidates
+            if candidate["inventory_type"] != "cloud":
+                continue
+
+            # Check if flavor mapping for current label_name already
+            # exists. This is an invalid condition.
+            if candidate.get("flavor_map") and candidate["flavor_map"].get(
+                    label_name):
+                error = True
+                LOG.error(_LE("Flavor mapping for label name {} already"
+                              "exists").format(label_name))
+                return {'response': None, 'error': error}
+
+            # RPC call to inventory provider for matching hpa capabilities
+            results = self.ip_ext_manager.map_method(
+                'match_hpa',
+                candidate=candidate,
+                features=features
+            )
+
+            if results and len(results) > 0:
+                flavor_info = results[0]
+            else:
+                flavor_info = None
+                LOG.info(
+                    _LW("No flavor mapping returned by "
+                        "inventory provider: {} for candidate: {}").format(
+                        self.ip_ext_manager.names()[0],
+                        candidate.get("candidate_id")))
+            if not flavor_info:
+                discard_set.add(candidate.get("candidate_id"))
+            else:
+                if not flavor_info.get("flavor-name"):
+                    discard_set.add(candidate.get("candidate_id"))
+                else:
+                    # Create flavor_map if not exist already
+                    if not candidate.get("flavor_map"):
+                        candidate["flavor_map"] = {}
+                    # Create flavor mapping for label_name to flavor
+                    flavor_name = flavor_info.get("flavor-name")
+                    candidate["flavor_map"][label_name] = flavor_name
+
+        # return candidates not in discard set
+        candidate_list[:] = [c for c in candidate_list
+                             if c['candidate_id'] not in discard_set]
+        LOG.info(_LI(
+            "Candidates with matching hpa capabilities: {}, "
+            "inventory provider: {}").format(candidate_list,
+                                             self.ip_ext_manager.names()[0]))
+        return {'response': candidate_list, 'error': error}
+
     def resolve_demands(self, ctx, arg):
 
         log_util.setLoggerFilter(LOG, ctx.get('keyspace'), ctx.get('plan_id'))

diff --git a/conductor/conductor/tests/unit/data/hpa_constraints.json b/conductor/conductor/tests/unit/data/hpa_constraints.json
new file mode 100644 (file)
index 0000000..3954cd5
--- /dev/null
+++ b/conductor/conductor/tests/unit/data/hpa_constraints.json
@@ -0,0 +1,171 @@
+{
+  "conductor_solver": {
+    "constraints": [
+      {
+        "hpa_constraint_vG": {
+          "demands": ["vG"],
+          "name": "hpa_constraint_vG",
+          "type": "hpa",
+          "properties": {
+            "evaluate": [
+              {
+                "features": [
+                  {
+                    "architecture": "generic",
+                    "hpa-feature": "basicCapabilities",
+                    "hpa-feature-attributes": [
+                      {
+                        "hpa-attribute-key": "numVirtualCpu",
+                        "hpa-attribute-value": "4",
+                        "operator": "="
+                      },
+                      {
+                        "hpa-attribute-key": "virtualMemSize",
+                        "hpa-attribute-value": "4",
+                        "operator": "=",
+                        "unit": "GB"
+                      }
+                    ],
+                    "hpa-version": "v1"
+                  },
+                  {
+                    "architecture": "generic",
+                    "hpa-feature": "numa",
+                    "hpa-feature-attributes": [
+                      {
+                        "hpa-attribute-key": "numaNodes",
+                        "hpa-attribute-value": "2",
+                        "operator": "="
+                      },
+                      {
+                        "hpa-attribute-key": "numaCpu-0",
+                        "hpa-attribute-value": "2",
+                        "operator": "="
+                      },
+                      {
+                        "hpa-attribute-key": "numaCpu-1",
+                        "hpa-attribute-value": "4",
+                        "operator": "="
+                      },
+                      {
+                        "hpa-attribute-key": "numaMem-0",
+                        "hpa-attribute-value": "2",
+                        "operator": "=",
+                        "unit": "GB"
+                      },
+                      {
+                        "hpa-attribute-key": "numaMem-1",
+                        "hpa-attribute-value": "4",
+                        "operator": "=",
+                        "unit": "GB"
+                      }
+                    ],
+                    "hpa-version": "v1"
+                  },
+                  {
+                    "architecture": "generic",
+                    "hpa-feature": "cpuPinning",
+                    "hpa-feature-attributes": [
+                      {
+                        "hpa-attribute-key": "logicalCpuThreadPinningPolicy",
+                        "hpa-attribute-value": "prefer",
+                        "operator": "="
+                      },
+                      {
+                        "hpa-attribute-key": "logicalCpuPinningPolicy",
+                        "hpa-attribute-value": "dedicated",
+                        "operator": "="
+                      }
+                    ],
+                    "hpa-version": "v1"
+                  }
+                ],
+                "label": "flavor_label_1"
+              },
+              {
+                "features": [
+                  {
+                    "architecture": "generic",
+                    "hpa-feature": "basicCapabilities",
+                    "hpa-feature-attributes": [
+                      {
+                        "hpa-attribute-key": "numVirtualCpu",
+                        "hpa-attribute-value": "8",
+                        "operator": "="
+                      },
+                      {
+                        "hpa-attribute-key": "virtualMemSize",
+                        "hpa-attribute-value": "16",
+                        "operator": "=",
+                        "unit": "GB"
+                      }
+                    ],
+                    "hpa-version": "v1"
+                  },
+                  {
+                    "architecture": "generic",
+                    "hpa-feature": "numa",
+                    "hpa-feature-attributes": [
+                      {
+                        "hpa-attribute-key": "numaNodes",
+                        "hpa-attribute-value": "2",
+                        "operator": "="
+                      },
+                      {
+                        "hpa-attribute-key": "numaCpu-0",
+                        "hpa-attribute-value": "2",
+                        "operator": "="
+                      },
+                      {
+                        "hpa-attribute-key": "numaCpu-1",
+                        "hpa-attribute-value": "4",
+                        "operator": "="
+                      },
+                      {
+                        "hpa-attribute-key": "numaMem-0",
+                        "hpa-attribute-value": "2",
+                        "operator": "=",
+                        "unit": "GB"
+                      },
+                      {
+                        "hpa-attribute-key": "numaMem-1",
+                        "hpa-attribute-value": "4",
+                        "operator": "=",
+                        "unit": "GB"
+                      }
+                    ],
+                    "hpa-version": "v1"
+                  },
+                  {
+                    "architecture": "generic",
+                    "hpa-feature": "memoryPageSize",
+                    "hpa-feature-attributes": [
+                      {
+                        "hpa-attribute-key": "memoryPageSize",
+                        "hpa-attribute-value": "2",
+                        "operator": "=",
+                        "unit": "GB"
+                      }
+                    ],
+                    "hpa-version": "v1"
+                  }
+                ],
+                "label": "flavor_label_2"
+              }
+            ]
+          }
+        }
+      },
+      {
+        "constraint_vgmux_customer": {
+          "type": "distance_to_location",
+          "demands": ["vGMuxInfra"],
+          "properties": {
+            "distance": "< 100 km",
+            "location": "customer_loc"
+          }
+        }
+      }
+    ]
+  }
+}
\ No newline at end of file

diff --git a/conductor/conductor/tests/unit/data/test_service.py b/conductor/conductor/tests/unit/data/test_service.py
index 89f1833..385b45d 100644 (file)
--- a/conductor/conductor/tests/unit/data/test_service.py
+++ b/conductor/conductor/tests/unit/data/test_service.py
@@ -16,6 +16,7 @@
 #
 # -------------------------------------------------------------------------
 #
+import copy
 import json
 import unittest
 import uuid
@@ -34,7 +35,6 @@ from oslo_config import cfg
 class TestDataEndpoint(unittest.TestCase):
 
     def setUp(self):
-        cfg.CONF.set_override('keyspace', 'conductor')
         ip_ext_manager = (
             ip_ext.Manager(cfg.CONF, 'conductor.inventory_provider.plugin'))
         sc_ext_manager = (
@@ -218,6 +218,69 @@ class TestDataEndpoint(unittest.TestCase):
         self.assertEqual(expected_response,
                          self.data_ep.resolve_demands(ctxt, req_json))
 
+    @mock.patch.object(service.LOG, 'error')
+    @mock.patch.object(service.LOG, 'info')
+    @mock.patch.object(stevedore.ExtensionManager, 'names')
+    @mock.patch.object(stevedore.ExtensionManager, 'map_method')
+    def test_get_candidates_with_hpa(self, hpa_mock, ext_mock1,
+                                     info_mock, error_mock):
+        req_json_file = './conductor/tests/unit/data/candidate_list.json'
+        hpa_json_file = './conductor/tests/unit/data/hpa_constraints.json'
+        hpa_json = yaml.safe_load(open(hpa_json_file).read())
+        req_json = yaml.safe_load(open(req_json_file).read())
+        candidate_list = req_json['candidate_list']
+        (constraint_id, constraint_info) = \
+            hpa_json["conductor_solver"]["constraints"][0].items()[0]
+        hpa_constraint = constraint_info['properties']
+        features = hpa_constraint['evaluate'][0]['features']
+        label_name = hpa_constraint['evaluate'][0]['label']
+        ext_mock1.return_value = ['aai']
+        flavor_info = {"flavor-id": "vim-flavor-id1",
+                       "flavor-name": "vim-flavor-name1"}
+        hpa_mock.return_value = [flavor_info]
+        self.maxDiff = None
+        args = generate_args(candidate_list, features, label_name)
+        hpa_candidate_list = copy.deepcopy(candidate_list)
+        hpa_candidate_list[1]['flavor_map'] = {}
+        hpa_candidate_list[1]['flavor_map'][label_name] = "vim-flavor-name1"
+        expected_response = {'response': hpa_candidate_list, 'error': False}
+        self.assertEqual(expected_response,
+                         self.data_ep.get_candidates_with_hpa(None, args))
+
+        hpa_candidate_list2 = list()
+        hpa_candidate_list2.append(copy.deepcopy(candidate_list[0]))
+        args = generate_args(candidate_list, features, label_name)
+        hpa_mock.return_value = []
+        expected_response = {'response': hpa_candidate_list2, 'error': False}
+        self.assertEqual(expected_response,
+                         self.data_ep.get_candidates_with_hpa(None, args))
+
+        flavor_info = {}
+        hpa_mock.return_value = [flavor_info]
+        expected_response = {'response': hpa_candidate_list2, 'error': False}
+        self.assertEqual(expected_response,
+                         self.data_ep.get_candidates_with_hpa(None, args))
+
+        flavor_info = {"flavor-id": "vim-flavor-id1",
+                       "flavor-name": ""}
+        hpa_mock.return_value = [flavor_info]
+        expected_response = {'response': hpa_candidate_list2, 'error': False}
+        self.assertEqual(expected_response,
+                         self.data_ep.get_candidates_with_hpa(None, args))
+
+        flavor_info = {"flavor-id": "vim-flavor-id1"}
+        hpa_mock.return_value = [flavor_info]
+        expected_response = {'response': hpa_candidate_list2, 'error': False}
+        self.assertEqual(expected_response,
+                         self.data_ep.get_candidates_with_hpa(None, args))
+
+
+def generate_args(candidate_list, features, label_name):
+    arg_candidate_list = copy.deepcopy(candidate_list)
+    args = {"candidate_list": arg_candidate_list,
+            "features": features,
+            "label_name": label_name}
+    return args
 
 def ip_ext_sideeffect(*args, **kwargs):
     req_json_file = './conductor/tests/unit/data/constraints.json'