HPA Enablement for A&AI data plugin

Implemented HPA Enablement feature support for A&AI data plugin
Added flavors and hpa-capability child nodes to cloud-region cache.
Added unit tests for _get_flavors
Modified unit tests for resolve_demands

Change-Id: Ifeedb5f53a351079e8adbd42d216633062c36e1b
Issue-ID: OPTFRA-167
Signed-off-by: Dileep Ranganathan <dileep.ranganathan@intel.com>

diff --git a/conductor/conductor/data/plugins/constants.py b/conductor/conductor/data/plugins/constants.py
new file mode 100644 (file)
index 0000000..8c0ca08
--- /dev/null
+++ b/conductor/conductor/data/plugins/constants.py
@@ -0,0 +1,24 @@
+#
+# -------------------------------------------------------------------------
+#   Copyright (c) 2018 Intel Corporation Intellectual Property
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+
+'''Constants used in data plugin'''
+
+CLOUD_REGIONS_URI = '/cloud-infrastructure/cloud-regions/cloud-region'
+
+FLAVORS_URI = CLOUD_REGIONS_URI + '/%s/%s/flavors/?depth=all'
\ No newline at end of file

diff --git a/conductor/conductor/data/plugins/inventory_provider/aai.py b/conductor/conductor/data/plugins/inventory_provider/aai.py
index f25557f..c40e15c 100644 (file)
--- a/conductor/conductor/data/plugins/inventory_provider/aai.py
+++ b/conductor/conductor/data/plugins/inventory_provider/aai.py
@@ -22,6 +22,7 @@ import time
 import uuid
 
 from conductor.common import rest
+from conductor.data.plugins import constants
 from conductor.data.plugins.inventory_provider import base
 from conductor.i18n import _LE, _LI
 from oslo_config import cfg
@@ -224,6 +225,9 @@ class AAI(base.InventoryProviderBase):
                 cloud_type = region.get('cloud-type')
                 cloud_zone = region.get('cloud-zone')
 
+                # Added for HPA support
+                flavors = self._get_flavors(cloud_owner, cloud_region_id)
+
                 physical_location_list = self._get_aai_rel_link_data(data = region, related_to = 'complex', search_key = 'complex.physical-location-id')
                 if len(physical_location_list) > 0:
                     physical_location_id = physical_location_list[0].get('d_value')
@@ -290,7 +294,8 @@ class AAI(base.InventoryProviderBase):
                         'state': state,
                         'region': region,
                         'country': country,
-                    }
+                    },
+                    'flavors': flavors
                 }
                 LOG.debug("Candidate with cloud_region_id '{}' selected "
                           "as a potential candidate - ".format(cloud_region_id))
@@ -409,6 +414,37 @@ class AAI(base.InventoryProviderBase):
         regions = self._aai_cache.get('cloud_region', {})
         return regions
 
+    def _get_flavors(self, cloud_owner, cloud_region_id):
+        '''
+        Fetch all flavors of a given cloud regions specified using
+        {cloud-owner}/{cloud-region-id} composite key
+        :return flavors_info json object which list of flavor nodes and
+        its children - HPACapabilities:
+        '''
+        LOG.debug("Fetch all flavors and its child nodes HPACapabilities")
+        flavor_path = constants.FLAVORS_URI % (cloud_owner, cloud_region_id)
+        path = self._aai_versioned_path(flavor_path)
+        LOG.debug("Flavors path '{}' ".format(path))
+
+        response = self._request(path=path, context="flavors", value="all")
+        if response is None:
+            return
+        if response.status_code == 200:
+            flavors_info = response.json()
+            if not flavors_info or not flavors_info["flavor"] or \
+                    len(flavors_info["flavor"]) == 0:
+                LOG.error(_LE("Flavor is missing in Cloud-Region {}/{}").
+                          format(cloud_owner, cloud_region_id))
+                return
+            LOG.debug(flavors_info)
+            # Remove extraneous flavor information
+            return flavors_info
+        else:
+            LOG.error(_LE("Received Error while fetching flavors from" \
+                          "Cloud-region {}/{}").format(cloud_owner,
+                                                       cloud_region_id))
+            return
+
     def _get_aai_path_from_link(self, link):
         path = link.split(self.version, 1)
         if not path or len(path) <= 1:
@@ -841,6 +877,10 @@ class AAI(base.InventoryProviderBase):
                         candidate['country'] = \
                             region['complex']['country']
 
+                        # Added for HPA
+                        candidate['flavors'] = \
+                            region['flavors']
+
                         if self.check_sriov_automation(
                                 candidate['cloud_region_version'], name,
                                 candidate['candidate_id']):

diff --git a/conductor/conductor/tests/unit/data/plugins/inventory_provider/_request_get_flavors.json b/conductor/conductor/tests/unit/data/plugins/inventory_provider/_request_get_flavors.json
new file mode 100644 (file)
index 0000000..9fd7dcc
--- /dev/null
+++ b/conductor/conductor/tests/unit/data/plugins/inventory_provider/_request_get_flavors.json
@@ -0,0 +1,195 @@
+{
+  "flavor": [
+    {
+      "flavor-id": "9cf8220b-4d96-4c30-a426-2e9382f3fff2",
+      "flavor-name": "flavor-numa-cpu-topology-instruction-set",
+      "flavor-vcpus": 64,
+      "flavor-ram": 65536,
+      "flavor-disk": 1048576,
+      "flavor-ephemeral": 128,
+      "flavor-swap": "0",
+      "flavor-is-public": false,
+      "flavor-selflink": "pXtX",
+      "flavor-disabled": false,
+      "hpa-capabilities": {
+        "hpa-capability": [
+          {
+            "hpa-capability-id": "13ec6d4d-7fee-48d8-9e4a-c598feb101ed",
+            "hpa-feature": "basicCapabilities",
+            "hpa-version": "v1",
+            "architecture": "generic",
+            "resource-version": "1520943845409",
+            "hpa-feature-attributes": [
+              {
+                "hpa-attribute-key": "numVirtualCpu",
+                "hpa-attribute-value": "{value:4}",
+                "resource-version": "1520943845416"
+              },
+              {
+                "hpa-attribute-key": "virtualMemSize",
+                "hpa-attribute-value": "{value:4, unit:\"GB\" }",
+                "resource-version": "1520943845427"
+              }
+            ]
+          },
+          {
+            "hpa-capability-id": "01a4bfe1-1993-4fda-bd1c-ef333b4f76a9",
+            "hpa-feature": "cpuInstructionSetExtensions",
+            "hpa-version": "v1",
+            "architecture": "Intel64",
+            "resource-version": "1520943846236",
+            "hpa-feature-attributes": [
+              {
+                "hpa-attribute-key": "instructionSetExtensions",
+                "hpa-attribute-value": "{value:{['AAA', 'BBB', 'CCC', 'DDD']}}",
+                "resource-version": "1520943846241"
+              }
+            ]
+          },
+          {
+            "hpa-capability-id": "167ad6a2-7d9c-4bf2-9a1b-30e5311b8c66",
+            "hpa-feature": "numa",
+            "hpa-version": "v1",
+            "architecture": "generic",
+            "resource-version": "1520943846158",
+            "hpa-feature-attributes": [
+              {
+                "hpa-attribute-key": "numaCpu-0",
+                "hpa-attribute-value": "{value:2}",
+                "resource-version": "1520943846178"
+              },
+              {
+                "hpa-attribute-key": "numaMem-0",
+                "hpa-attribute-value": "{value:2, unit:\"GB\" }",
+                "resource-version": "1520943846204"
+              },
+              {
+                "hpa-attribute-key": "numaCpu-1",
+                "hpa-attribute-value": "{value:4}",
+                "resource-version": "1520943846191"
+              },
+              {
+                "hpa-attribute-key": "numaMem-1",
+                "hpa-attribute-value": "{value:4, unit:\"GB\" }",
+                "resource-version": "1520943846217"
+              },
+              {
+                "hpa-attribute-key": "numaNodes",
+                "hpa-attribute-value": "{value:2}",
+                "resource-version": "1520943846163"
+              }
+            ]
+          },
+          {
+            "hpa-capability-id": "8fa22e64-41b4-471f-96ad-6c4708635e4c",
+            "hpa-feature": "cpuTopology",
+            "hpa-version": "v1",
+            "architecture": "generic",
+            "resource-version": "1520943845443",
+            "hpa-feature-attributes": [
+              {
+                "hpa-attribute-key": "numCpuCores",
+                "hpa-attribute-value": "{value:8}",
+                "resource-version": "1520943845456"
+              },
+              {
+                "hpa-attribute-key": "numCpuSockets",
+                "hpa-attribute-value": "{value:6}",
+                "resource-version": "1520943845447"
+              },
+              {
+                "hpa-attribute-key": "numCpuThreads",
+                "hpa-attribute-value": "{value:8}",
+                "resource-version": "1520943846129"
+              }
+            ]
+          }
+        ]
+      },
+      "resource-version": "1520943845399"
+    },
+    {
+      "flavor-id": "f5aa2b2e-3206-41b6-80d5-cf041b098c43",
+      "flavor-name": "flavor-cpu-pinning-ovsdpdk-instruction-set",
+      "flavor-vcpus": 32,
+      "flavor-ram": 131072,
+      "flavor-disk": 2097152,
+      "flavor-ephemeral": 128,
+      "flavor-swap": "0",
+      "flavor-is-public": false,
+      "flavor-selflink": "pXtX",
+      "flavor-disabled": false,
+      "hpa-capabilities": {
+        "hpa-capability": [
+          {
+            "hpa-capability-id": "8d36a8fe-bfee-446a-bbcb-881ee66c8f78",
+            "hpa-feature": "ovsDpdk",
+            "hpa-version": "v1",
+            "architecture": "generic",
+            "resource-version": "1520943846328",
+            "hpa-feature-attributes": [
+              {
+                "hpa-attribute-key": "dataProcessingAccelerationLibrary",
+                "hpa-attribute-value": "{value:\"v18.02\"}",
+                "resource-version": "1520943846346"
+              }
+            ]
+          },
+          {
+            "hpa-capability-id": "c140c945-1532-4908-86c9-d7f71416f1dd",
+            "hpa-feature": "cpuPinning",
+            "hpa-version": "v1",
+            "architecture": "generic",
+            "resource-version": "1520943846297",
+            "hpa-feature-attributes": [
+              {
+                "hpa-attribute-key": "logicalCpuPinningPolicy",
+                "hpa-attribute-value": "{value:\"dedicated\"}",
+                "resource-version": "1520943846312"
+              },
+              {
+                "hpa-attribute-key": "logicalCpuThreadPinningPolicy",
+                "hpa-attribute-value": "{value:\"prefer\"}",
+                "resource-version": "1520943846301"
+              }
+            ]
+          },
+          {
+            "hpa-capability-id": "4d04f4d8-e257-4442-8417-19a525e56096",
+            "hpa-feature": "cpuInstructionSetExtensions",
+            "hpa-version": "v1",
+            "architecture": "generic",
+            "resource-version": "1520943846362",
+            "hpa-feature-attributes": [
+              {
+                "hpa-attribute-key": "instructionSetExtensions",
+                "hpa-attribute-value": "{value:{['A11', 'B22']}}",
+                "resource-version": "1520943846365"
+              }
+            ]
+          },
+          {
+            "hpa-capability-id": "4565615b-1077-4bb5-a340-c5be48db2aaa",
+            "hpa-feature": "basicCapabilities",
+            "hpa-version": "v1",
+            "architecture": "generic",
+            "resource-version": "1520943846269",
+            "hpa-feature-attributes": [
+              {
+                "hpa-attribute-key": "virtualMemSize",
+                "hpa-attribute-value": "{value:16, unit:\"GB\" }",
+                "resource-version": "1520943846282"
+              },
+              {
+                "hpa-attribute-key": "numVirtualCpu",
+                "hpa-attribute-value": "{value:8}",
+                "resource-version": "1520943846272"
+              }
+            ]
+          }
+        ]
+      },
+      "resource-version": "1520943846264"
+    }
+  ]
+}
\ No newline at end of file

diff --git a/conductor/conductor/tests/unit/data/plugins/inventory_provider/regions.json b/conductor/conductor/tests/unit/data/plugins/inventory_provider/regions.json
index e6412c1..d05a7fd 100644 (file)
--- a/conductor/conductor/tests/unit/data/plugins/inventory_provider/regions.json
+++ b/conductor/conductor/tests/unit/data/plugins/inventory_provider/regions.json
@@ -1,20 +1,215 @@
 {
-    "region-name": {
-        "cloud_type": "opensource",
-        "complex": {
-            "city": "Middletown",
-            "state": "NJ",
-            "longitude": "30.12",
-            "latitude": "50.34",
-            "country": "USA",
-            "complex_name": "complex-name",
-            "region": "USA",
-            "complex_id": "complex-id"
+  "region-name": {
+    "cloud_type": "opensource",
+    "complex": {
+      "city": "Middletown",
+      "state": "NJ",
+      "longitude": "30.12",
+      "latitude": "50.34",
+      "country": "USA",
+      "complex_name": "complex-name",
+      "region": "USA",
+      "complex_id": "complex-id"
+    },
+    "cloud_region_version": "1.0",
+    "physical_location_id": "location-id",
+    "cloud_owner": "cloud-owner",
+    "cloud_zone": "cloud-zone",
+    "complex_name": "complex-name",
+    "flavors": {
+      "flavor": [
+        {
+          "flavor-id": "9cf8220b-4d96-4c30-a426-2e9382f3fff2",
+          "flavor-name": "flavor-numa-cpu-topology-instruction-set",
+          "flavor-vcpus": 64,
+          "flavor-ram": 65536,
+          "flavor-disk": 1048576,
+          "flavor-ephemeral": 128,
+          "flavor-swap": "0",
+          "flavor-is-public": false,
+          "flavor-selflink": "pXtX",
+          "flavor-disabled": false,
+          "hpa-capabilities": {
+            "hpa-capability": [
+              {
+                "hpa-capability-id": "13ec6d4d-7fee-48d8-9e4a-c598feb101ed",
+                "hpa-feature": "basicCapabilities",
+                "hpa-version": "v1",
+                "architecture": "generic",
+                "resource-version": "1520943845409",
+                "hpa-feature-attributes": [
+                  {
+                    "hpa-attribute-key": "numVirtualCpu",
+                    "hpa-attribute-value": "{value:4}",
+                    "resource-version": "1520943845416"
+                  },
+                  {
+                    "hpa-attribute-key": "virtualMemSize",
+                    "hpa-attribute-value": "{value:4, unit:\"GB\" }",
+                    "resource-version": "1520943845427"
+                  }
+                ]
+              },
+              {
+                "hpa-capability-id": "01a4bfe1-1993-4fda-bd1c-ef333b4f76a9",
+                "hpa-feature": "cpuInstructionSetExtensions",
+                "hpa-version": "v1",
+                "architecture": "Intel64",
+                "resource-version": "1520943846236",
+                "hpa-feature-attributes": [
+                  {
+                    "hpa-attribute-key": "instructionSetExtensions",
+                    "hpa-attribute-value": "{value:{['AAA', 'BBB', 'CCC', 'DDD']}}",
+                    "resource-version": "1520943846241"
+                  }
+                ]
+              },
+              {
+                "hpa-capability-id": "167ad6a2-7d9c-4bf2-9a1b-30e5311b8c66",
+                "hpa-feature": "numa",
+                "hpa-version": "v1",
+                "architecture": "generic",
+                "resource-version": "1520943846158",
+                "hpa-feature-attributes": [
+                  {
+                    "hpa-attribute-key": "numaCpu-0",
+                    "hpa-attribute-value": "{value:2}",
+                    "resource-version": "1520943846178"
+                  },
+                  {
+                    "hpa-attribute-key": "numaMem-0",
+                    "hpa-attribute-value": "{value:2, unit:\"GB\" }",
+                    "resource-version": "1520943846204"
+                  },
+                  {
+                    "hpa-attribute-key": "numaCpu-1",
+                    "hpa-attribute-value": "{value:4}",
+                    "resource-version": "1520943846191"
+                  },
+                  {
+                    "hpa-attribute-key": "numaMem-1",
+                    "hpa-attribute-value": "{value:4, unit:\"GB\" }",
+                    "resource-version": "1520943846217"
+                  },
+                  {
+                    "hpa-attribute-key": "numaNodes",
+                    "hpa-attribute-value": "{value:2}",
+                    "resource-version": "1520943846163"
+                  }
+                ]
+              },
+              {
+                "hpa-capability-id": "8fa22e64-41b4-471f-96ad-6c4708635e4c",
+                "hpa-feature": "cpuTopology",
+                "hpa-version": "v1",
+                "architecture": "generic",
+                "resource-version": "1520943845443",
+                "hpa-feature-attributes": [
+                  {
+                    "hpa-attribute-key": "numCpuCores",
+                    "hpa-attribute-value": "{value:8}",
+                    "resource-version": "1520943845456"
+                  },
+                  {
+                    "hpa-attribute-key": "numCpuSockets",
+                    "hpa-attribute-value": "{value:6}",
+                    "resource-version": "1520943845447"
+                  },
+                  {
+                    "hpa-attribute-key": "numCpuThreads",
+                    "hpa-attribute-value": "{value:8}",
+                    "resource-version": "1520943846129"
+                  }
+                ]
+              }
+            ]
+          },
+          "resource-version": "1520943845399"
         },
-        "cloud_region_version": "1.0",
-        "physical_location_id": "location-id",
-        "cloud_owner": "cloud-owner",
-        "cloud_zone": "cloud-zone",
-        "complex_name": "complex-name"
+        {
+          "flavor-id": "f5aa2b2e-3206-41b6-80d5-cf041b098c43",
+          "flavor-name": "flavor-cpu-pinning-ovsdpdk-instruction-set",
+          "flavor-vcpus": 32,
+          "flavor-ram": 131072,
+          "flavor-disk": 2097152,
+          "flavor-ephemeral": 128,
+          "flavor-swap": "0",
+          "flavor-is-public": false,
+          "flavor-selflink": "pXtX",
+          "flavor-disabled": false,
+          "hpa-capabilities": {
+            "hpa-capability": [
+              {
+                "hpa-capability-id": "8d36a8fe-bfee-446a-bbcb-881ee66c8f78",
+                "hpa-feature": "ovsDpdk",
+                "hpa-version": "v1",
+                "architecture": "generic",
+                "resource-version": "1520943846328",
+                "hpa-feature-attributes": [
+                  {
+                    "hpa-attribute-key": "dataProcessingAccelerationLibrary",
+                    "hpa-attribute-value": "{value:\"v18.02\"}",
+                    "resource-version": "1520943846346"
+                  }
+                ]
+              },
+              {
+                "hpa-capability-id": "c140c945-1532-4908-86c9-d7f71416f1dd",
+                "hpa-feature": "cpuPinning",
+                "hpa-version": "v1",
+                "architecture": "generic",
+                "resource-version": "1520943846297",
+                "hpa-feature-attributes": [
+                  {
+                    "hpa-attribute-key": "logicalCpuPinningPolicy",
+                    "hpa-attribute-value": "{value:\"dedicated\"}",
+                    "resource-version": "1520943846312"
+                  },
+                  {
+                    "hpa-attribute-key": "logicalCpuThreadPinningPolicy",
+                    "hpa-attribute-value": "{value:\"prefer\"}",
+                    "resource-version": "1520943846301"
+                  }
+                ]
+              },
+              {
+                "hpa-capability-id": "4d04f4d8-e257-4442-8417-19a525e56096",
+                "hpa-feature": "cpuInstructionSetExtensions",
+                "hpa-version": "v1",
+                "architecture": "generic",
+                "resource-version": "1520943846362",
+                "hpa-feature-attributes": [
+                  {
+                    "hpa-attribute-key": "instructionSetExtensions",
+                    "hpa-attribute-value": "{value:{['A11', 'B22']}}",
+                    "resource-version": "1520943846365"
+                  }
+                ]
+              },
+              {
+                "hpa-capability-id": "4565615b-1077-4bb5-a340-c5be48db2aaa",
+                "hpa-feature": "basicCapabilities",
+                "hpa-version": "v1",
+                "architecture": "generic",
+                "resource-version": "1520943846269",
+                "hpa-feature-attributes": [
+                  {
+                    "hpa-attribute-key": "virtualMemSize",
+                    "hpa-attribute-value": "{value:16, unit:\"GB\" }",
+                    "resource-version": "1520943846282"
+                  },
+                  {
+                    "hpa-attribute-key": "numVirtualCpu",
+                    "hpa-attribute-value": "{value:8}",
+                    "resource-version": "1520943846272"
+                  }
+                ]
+              }
+            ]
+          },
+          "resource-version": "1520943846264"
+        }
+      ]
     }
+  }
 }
\ No newline at end of file

diff --git a/conductor/conductor/tests/unit/data/plugins/inventory_provider/test_aai.py b/conductor/conductor/tests/unit/data/plugins/inventory_provider/test_aai.py
index 6b985a5..b148579 100644 (file)
--- a/conductor/conductor/tests/unit/data/plugins/inventory_provider/test_aai.py
+++ b/conductor/conductor/tests/unit/data/plugins/inventory_provider/test_aai.py
@@ -138,6 +138,7 @@ class TestAAI(unittest.TestCase):
         self.mock_get_complex = mock.patch.object(AAI, '_get_complex', return_value=complex_json)
         self.mock_get_complex.start()
 
+        flavor_info = regions_response["region-name"]["flavors"]
         self.maxDiff = None
         self.assertEqual({u'demand_name': [
             {'candidate_id': u'service-instance-id', 'city': None,
@@ -161,8 +162,9 @@ class TestAAI(unittest.TestCase):
              'location_type': 'att_aic', 'longitude': u'30.12',
              'physical_location_id': u'complex-id', 'region': u'USA',
              'service_resource_id': u'service-resource-id-123',
-             'sriov_automation': 'false', 'state': u'NJ'}]},
-                         self.aai_ep.resolve_demands(demands_list))
+             'sriov_automation': 'false', 'state': u'NJ',
+             'flavors': flavor_info}]},
+            self.aai_ep.resolve_demands(demands_list))
 
     def test_get_complex(self):
 
@@ -244,6 +246,9 @@ class TestAAI(unittest.TestCase):
         complex_json_file = './conductor/tests/unit/data/plugins/inventory_provider/_cached_complex.json'
         complex_json = json.loads(open(complex_json_file).read())
 
+        flavors_json_file = './conductor/tests/unit/data/plugins/inventory_provider/_request_get_flavors.json'
+        flavors_json = json.loads(open(flavors_json_file).read())
+
         response = mock.MagicMock()
         response.status_code = 200
         response.ok = True
@@ -255,6 +260,10 @@ class TestAAI(unittest.TestCase):
         self.mock_get_complex = mock.patch.object(AAI, '_get_complex', return_value=complex_json)
         self.mock_get_complex.start()
 
+        self.mock_get_flavors = mock.patch.object(AAI, '_get_flavors',
+                                                  return_value=flavors_json)
+        self.mock_get_flavors.start()
+
         self.assertEqual(None,
                          self.aai_ep._refresh_cache())
 
@@ -265,4 +274,27 @@ class TestAAI(unittest.TestCase):
         related_to = "service-instance"
 
         self.assertEqual("relationship-link",
-                         self.aai_ep._get_aai_rel_link(relatonship_response, related_to))
\ No newline at end of file
+                         self.aai_ep._get_aai_rel_link(relatonship_response, related_to))
+
+    def test_get_flavor(self):
+        flavors_json_file = './conductor/tests/unit/data/plugins/inventory_provider/_request_get_flavors.json'
+        flavors_json = json.loads(open(flavors_json_file).read())
+
+        response = mock.MagicMock()
+        response.json.return_value = None
+
+        self.mock_get_request = mock.patch.object(AAI, '_request',
+                                                  return_value=response)
+        self.mock_get_request.start()
+
+        flavors_info = self.aai_ep._get_flavors("mock-cloud-owner",
+                                                "mock-cloud-region-id")
+        self.assertEqual(None, flavors_info)
+
+        response.status_code = 200
+        response.ok = True
+        response.json.return_value = flavors_json
+
+        flavors_info = self.aai_ep._get_flavors("mock-cloud-owner",
+                                                "mock-cloud-region-id")
+        self.assertEqual(2, len(flavors_info['flavor']))
\ No newline at end of file