azure/multicloud_azure/pub/utils/restcall.py

   1 # Copyright (c) 2018 Amdocs
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at:
   6 #
   7 #       http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12
  13 import sys
  14 import traceback
  15 import logging
  16 import urllib2
  17 import uuid
  18 import httplib2
  19 import json
  20
  21 from multicloud_azure.pub.config.config import AAI_SCHEMA_VERSION
  22 from multicloud_azure.pub.config.config import AAI_SERVICE_URL
  23 from multicloud_azure.pub.config.config import AAI_USERNAME
  24 from multicloud_azure.pub.config.config import AAI_PASSWORD
  25 from multicloud_azure.pub.config.config import MSB_SERVICE_IP, MSB_SERVICE_PORT
  26
  27 from multicloud_azure.pub.exceptions import VimDriverAzureException
  28
  29 rest_no_auth, rest_oneway_auth, rest_bothway_auth = 0, 1, 2
  30 HTTP_200_OK, HTTP_201_CREATED = '200', '201'
  31 HTTP_204_NO_CONTENT, HTTP_202_ACCEPTED = '204', '202'
  32 status_ok_list = [HTTP_200_OK, HTTP_201_CREATED,
  33                   HTTP_204_NO_CONTENT, HTTP_202_ACCEPTED]
  34 HTTP_404_NOTFOUND, HTTP_403_FORBIDDEN = '404', '403'
  35 HTTP_401_UNAUTHORIZED, HTTP_400_BADREQUEST = '401', '400'
  36
  37 logger = logging.getLogger(__name__)
  38
  39
  40 def call_req(base_url, user, passwd, auth_type, resource, method, content='',
  41              headers=None):
  42     callid = str(uuid.uuid1())
  43 #    logger.debug("[%s]call_req('%s','%s','%s',%s,'%s','%s','%s')" % (
  44 #        callid, base_url, user, passwd, auth_type, resource, method, content))
  45     ret = None
  46     resp_status = ''
  47     resp = ""
  48     full_url = ""
  49
  50     try:
  51         full_url = combine_url(base_url, resource)
  52         if headers is None:
  53             headers = {}
  54             headers['content-type'] = 'application/json'
  55
  56         if user:
  57             headers['Authorization'] = 'Basic ' + \
  58                 ('%s:%s' % (user, passwd)).encode("base64")
  59         ca_certs = None
  60         for retry_times in range(3):
  61             http = httplib2.Http(
  62                 ca_certs=ca_certs,
  63                 disable_ssl_certificate_validation=(
  64                     auth_type == rest_no_auth))
  65             http.follow_all_redirects = True
  66             try:
  67                 logger.debug("request=%s" % full_url)
  68                 resp, resp_content = http.request(
  69                     full_url, method=method.upper(), body=content,
  70                     headers=headers)
  71                 resp_status = resp['status']
  72                 resp_body = resp_content.decode('UTF-8')
  73
  74                 if resp_status in status_ok_list:
  75                     ret = [0, resp_body, resp_status, resp]
  76                 else:
  77                     ret = [1, resp_body, resp_status, resp]
  78                 break
  79             except Exception as ex:
  80                 if 'httplib.ResponseNotReady' in str(sys.exc_info()):
  81                     logger.error(traceback.format_exc())
  82                     ret = [1, "Unable to connect to %s" % full_url,
  83                            resp_status, resp]
  84                     continue
  85                 raise ex
  86     except urllib2.URLError as err:
  87         ret = [2, str(err), resp_status, resp]
  88     except Exception as ex:
  89         logger.error(traceback.format_exc())
  90         logger.error("[%s]ret=%s" % (callid, str(sys.exc_info())))
  91         res_info = str(sys.exc_info())
  92         if 'httplib.ResponseNotReady' in res_info:
  93             res_info = ("The URL[%s] request failed or is not responding." %
  94                         full_url)
  95         ret = [3, res_info, resp_status, resp]
  96 #    logger.debug("[%s]ret=%s" % (callid, str(ret)))
  97     return ret
  98
  99
 100 def req_by_msb(resource, method, content=''):
 101     base_url = "http://%s:%s/" % (MSB_SERVICE_IP, MSB_SERVICE_PORT)
 102     return call_req(base_url, "", "", rest_no_auth, resource, method, content)
 103
 104
 105 def combine_url(base_url, resource):
 106     full_url = None
 107     if base_url.endswith('/') and resource.startswith('/'):
 108         full_url = base_url[:-1] + resource
 109     elif base_url.endswith('/') and not resource.startswith('/'):
 110         full_url = base_url + resource
 111     elif not base_url.endswith('/') and resource.startswith('/'):
 112         full_url = base_url + resource
 113     else:
 114         full_url = base_url + '/' + resource
 115     return full_url
 116
 117
 118 def get_res_from_aai(resource, content=''):
 119     headers = {
 120         'X-FromAppId': 'MultiCloud',
 121         'X-TransactionId': '9001',
 122         'content-type': 'application/json',
 123         'accept': 'application/json'
 124     }
 125     base_url = "%s/%s" % (AAI_SERVICE_URL, AAI_SCHEMA_VERSION)
 126     return call_req(base_url, AAI_USERNAME, AAI_PASSWORD, rest_no_auth,
 127                     resource, "GET", content, headers)
 128
 129
 130 class AAIClient(object):
 131     def __init__(self, cloud_owner, cloud_region):
 132         self.base_url = "%s/%s" % (AAI_SERVICE_URL, AAI_SCHEMA_VERSION)
 133         self.username = AAI_USERNAME
 134         self.password = AAI_PASSWORD
 135         self.default_headers = {
 136             'X-FromAppId': 'multicloud-azure',
 137             'X-TransactionId': '9004',
 138             'content-type': 'application/json',
 139             'accept': 'application/json'
 140         }
 141         self.cloud_owner = cloud_owner
 142         self.cloud_region = cloud_region
 143         self._vim_info = None
 144
 145     def get_vim(self, get_all=False):
 146         resource = ("/cloud-infrastructure/cloud-regions/cloud-region"
 147                     "/%s/%s" % (self.cloud_owner, self.cloud_region))
 148         if get_all:
 149             resource = "%s?depth=all" % resource
 150         resp = call_req(self.base_url, self.username, self.password,
 151                         rest_no_auth, resource, "GET",
 152                         headers=self.default_headers)
 153         if resp[0] != 0:
 154             raise VimDriverAzureException(
 155                 status_code=404,
 156                 content="Failed to query VIM with id (%s_%s) from extsys." % (
 157                     self.cloud_owner, self.cloud_region))
 158         return json.loads(resp[1])
 159
 160     def delete_vim(self):
 161         resp = self.get_vim(get_all=True)
 162         logger.debug('Delete cloud region')
 163         resource = ("/cloud-infrastructure/cloud-regions/cloud-region"
 164                     "/%s/%s?resource-version=%s" %
 165                     (self.cloud_owner, self.cloud_region,
 166                      resp['resource-version']))
 167         resp = call_req(self.base_url, self.username, self.password,
 168                         rest_no_auth, resource, "DELETE",
 169                         headers=self.default_headers)
 170         if resp[0] != 0:
 171             raise VimDriverAzureException(
 172                 status_code=400,
 173                 content="Failed to delete cloud %s_%s: %s." % (
 174                     self.cloud_owner, self.cloud_region, resp[1]))
 175
 176     def update_vim(self, content):
 177         self.add_flavors(content)
 178
 179     def update_identity_url(self):
 180         vim = self.get_vim()
 181         vim['identity-url'] = ("http://%s/api/multicloud/v0/%s_%s/identity/"
 182                                "v3" % (MSB_SERVICE_IP, self.cloud_owner,
 183                                        self.cloud_region))
 184         resource = ("/cloud-infrastructure/cloud-regions/cloud-region"
 185                     "/%s/%s" % (self.cloud_owner, self.cloud_region))
 186         logger.debug("Updating identity url %s" % vim)
 187         call_req(self.base_url, self.username, self.password,
 188                  rest_no_auth, resource, "PUT",
 189                  content=json.dumps(vim),
 190                  headers=self.default_headers)
 191
 192     def add_flavors(self, content):
 193         for flavor in content['flavors']:
 194             resource = ("/cloud-infrastructure/cloud-regions/cloud-region/"
 195                         "%s/%s/flavors/flavor/%s" % (
 196                             self.cloud_owner, self.cloud_region,
 197                             flavor['name']))
 198             body = {
 199                 'flavor-name': flavor['name'],
 200                 'flavor-vcpus': flavor['vcpus'],
 201                 'flavor-ram': flavor['ram'],
 202                 'flavor-disk': flavor['disk'],
 203                 'flavor-selflink': ""
 204             }
 205             # Handle extra specs
 206             if flavor['name'].startswith("onap."):
 207                 hpa_capabilities = self._get_hpa_capabilities(
 208                     flavor)
 209                 body['hpa-capabilities'] = {
 210                     'hpa-capability': hpa_capabilities}
 211
 212             logger.debug("Adding flavors to cloud region")
 213             call_req(self.base_url, self.username, self.password,
 214                      rest_no_auth, resource, "PUT",
 215                      content=json.dumps(body),
 216                      headers=self.default_headers)
 217
 218     def _get_hpa_capabilities(self, flavor):
 219         hpa_caps = []
 220
 221         # Basic capabilties
 222         caps_dict = self._get_hpa_basic_capabilities(flavor)
 223         if len(caps_dict) > 0:
 224             logger.debug("basic_capabilities_info: %s" % caps_dict)
 225             hpa_caps.append(caps_dict)
 226
 227         # cpupining capabilities
 228         caps_dict = self._get_cpupinning_capabilities(flavor['extra_specs'])
 229         if len(caps_dict) > 0:
 230             logger.debug("cpupining_capabilities_info: %s" % caps_dict)
 231             hpa_caps.append(caps_dict)
 232
 233         # cputopology capabilities
 234         caps_dict = self._get_cputopology_capabilities(flavor['extra_specs'])
 235         if len(caps_dict) > 0:
 236             logger.debug("cputopology_capabilities_info: %s" % caps_dict)
 237             hpa_caps.append(caps_dict)
 238
 239         # hugepages capabilities
 240         caps_dict = self._get_hugepages_capabilities(flavor['extra_specs'])
 241         if len(caps_dict) > 0:
 242             logger.debug("hugepages_capabilities_info: %s" % caps_dict)
 243             hpa_caps.append(caps_dict)
 244
 245         # numa capabilities
 246         caps_dict = self._get_numa_capabilities(flavor['extra_specs'])
 247         if len(caps_dict) > 0:
 248             logger.debug("numa_capabilities_info: %s" % caps_dict)
 249             hpa_caps.append(caps_dict)
 250
 251         # storage capabilities
 252         caps_dict = self._get_storage_capabilities(flavor)
 253         if len(caps_dict) > 0:
 254             logger.debug("storage_capabilities_info: %s" % caps_dict)
 255             hpa_caps.append(caps_dict)
 256
 257         # CPU instruction set extension capabilities
 258         caps_dict = self._get_instruction_set_capabilities(
 259             flavor['extra_specs'])
 260         if len(caps_dict) > 0:
 261             logger.debug("instruction_set_capabilities_info: %s" % caps_dict)
 262             hpa_caps.append(caps_dict)
 263
 264         # PCI passthrough capabilities
 265         caps_dict = self._get_pci_passthrough_capabilities(
 266             flavor['extra_specs'])
 267         if len(caps_dict) > 0:
 268             logger.debug("pci_passthrough_capabilities_info: %s" % caps_dict)
 269             hpa_caps.append(caps_dict)
 270
 271         # ovsdpdk capabilities
 272         caps_dict = self._get_ovsdpdk_capabilities()
 273         if len(caps_dict) > 0:
 274             logger.debug("ovsdpdk_capabilities_info: %s" % caps_dict)
 275             hpa_caps.append(caps_dict)
 276
 277         return hpa_caps
 278
 279     def _get_hpa_basic_capabilities(self, flavor):
 280         basic_capability = {}
 281         feature_uuid = uuid.uuid4()
 282
 283         basic_capability['hpa-capability-id'] = str(feature_uuid)
 284         basic_capability['hpa-feature'] = 'basicCapabilities'
 285         basic_capability['architecture'] = 'generic'
 286         basic_capability['hpa-version'] = 'v1'
 287
 288         basic_capability['hpa-feature-attributes'] = []
 289         basic_capability['hpa-feature-attributes'].append({
 290             'hpa-attribute-key': 'numVirtualCpu',
 291             'hpa-attribute-value': json.dumps(
 292                 {'value': str(flavor['vcpus'])})})
 293         basic_capability['hpa-feature-attributes'].append({
 294             'hpa-attribute-key': 'virtualMemSize',
 295             'hpa-attribute-value': json.dumps({'value': str(
 296                 flavor['ram']), 'unit': 'GB'})})
 297
 298         return basic_capability
 299
 300     def _get_cpupinning_capabilities(self, extra_specs):
 301         cpupining_capability = {}
 302         feature_uuid = uuid.uuid4()
 303
 304         if (extra_specs.get('hw:cpu_policy') or
 305                 extra_specs.get('hw:cpu_thread_policy')):
 306             cpupining_capability['hpa-capability-id'] = str(feature_uuid)
 307             cpupining_capability['hpa-feature'] = 'cpuPinning'
 308             cpupining_capability['architecture'] = 'generic'
 309             cpupining_capability['hpa-version'] = 'v1'
 310
 311             cpupining_capability['hpa-feature-attributes'] = []
 312             if extra_specs.get('hw:cpu_thread_policy'):
 313                 cpupining_capability['hpa-feature-attributes'].append({
 314                     'hpa-attribute-key': 'logicalCpuThreadPinningPolicy',
 315                     'hpa-attribute-value': json.dumps({'value': str(
 316                         extra_specs['hw:cpu_thread_policy'])})})
 317             if extra_specs.get('hw:cpu_policy'):
 318                 cpupining_capability['hpa-feature-attributes'].append({
 319                     'hpa-attribute-key': 'logicalCpuPinningPolicy',
 320                     'hpa-attribute-value': json.dumps({'value': str(
 321                         extra_specs['hw:cpu_policy'])})})
 322
 323         return cpupining_capability
 324
 325     def _get_cputopology_capabilities(self, extra_specs):
 326         cputopology_capability = {}
 327         feature_uuid = uuid.uuid4()
 328
 329         if (extra_specs.get('hw:cpu_sockets') or
 330                 extra_specs.get('hw:cpu_cores') or
 331                 extra_specs.get('hw:cpu_threads')):
 332             cputopology_capability['hpa-capability-id'] = str(feature_uuid)
 333             cputopology_capability['hpa-feature'] = 'cpuTopology'
 334             cputopology_capability['architecture'] = 'generic'
 335             cputopology_capability['hpa-version'] = 'v1'
 336
 337             cputopology_capability['hpa-feature-attributes'] = []
 338             if extra_specs.get('hw:cpu_sockets'):
 339                 cputopology_capability['hpa-feature-attributes'].append({
 340                     'hpa-attribute-key': 'numCpuSockets',
 341                     'hpa-attribute-value': json.dumps({'value': str(
 342                         extra_specs['hw:cpu_sockets'])})})
 343             if extra_specs.get('hw:cpu_cores'):
 344                 cputopology_capability['hpa-feature-attributes'].append({
 345                     'hpa-attribute-key': 'numCpuCores',
 346                     'hpa-attribute-value': json.dumps({'value': str(
 347                         extra_specs['hw:cpu_cores'])})})
 348             if extra_specs.get('hw:cpu_threads'):
 349                 cputopology_capability['hpa-feature-attributes'].append({
 350                     'hpa-attribute-key': 'numCpuThreads',
 351                     'hpa-attribute-value': json.dumps({'value': str(
 352                         extra_specs['hw:cpu_threads'])})})
 353
 354         return cputopology_capability
 355
 356     def _get_hugepages_capabilities(self, extra_specs):
 357         hugepages_capability = {}
 358         feature_uuid = uuid.uuid4()
 359
 360         if extra_specs.get('hw:mem_page_size'):
 361             hugepages_capability['hpa-capability-id'] = str(feature_uuid)
 362             hugepages_capability['hpa-feature'] = 'hugePages'
 363             hugepages_capability['architecture'] = 'generic'
 364             hugepages_capability['hpa-version'] = 'v1'
 365
 366             hugepages_capability['hpa-feature-attributes'] = []
 367             if extra_specs['hw:mem_page_size'] == 'large':
 368                 hugepages_capability['hpa-feature-attributes'].append({
 369                     'hpa-attribute-key': 'memoryPageSize',
 370                     'hpa-attribute-value': json.dumps(
 371                         {'value': '2', 'unit': 'MB'})})
 372             elif extra_specs['hw:mem_page_size'] == 'small':
 373                 hugepages_capability['hpa-feature-attributes'].append({
 374                     'hpa-attribute-key': 'memoryPageSize',
 375                     'hpa-attribute-value': json.dumps(
 376                         {'value': '4', 'unit': 'KB'})})
 377             elif extra_specs['hw:mem_page_size'] == 'any':
 378                 logger.info("Currently HPA feature memoryPageSize "
 379                             "did not support 'any' page!!")
 380             else:
 381                 hugepages_capability['hpa-feature-attributes'].append({
 382                     'hpa-attribute-key': 'memoryPageSize',
 383                     'hpa-attribute-value': json.dumps({'value': str(
 384                         extra_specs['hw:mem_page_size']), 'unit': 'KB'})
 385                     })
 386
 387         return hugepages_capability
 388
 389     def _get_numa_capabilities(self, extra_specs):
 390         numa_capability = {}
 391         feature_uuid = uuid.uuid4()
 392
 393         if extra_specs.get('hw:numa_nodes'):
 394             numa_capability['hpa-capability-id'] = str(feature_uuid)
 395             numa_capability['hpa-feature'] = 'numa'
 396             numa_capability['architecture'] = 'generic'
 397             numa_capability['hpa-version'] = 'v1'
 398
 399             numa_capability['hpa-feature-attributes'] = []
 400             numa_capability['hpa-feature-attributes'].append({
 401                 'hpa-attribute-key': 'numaNodes',
 402                 'hpa-attribute-value': json.dumps({'value': str(
 403                     extra_specs['hw:numa_nodes'])})
 404                 })
 405
 406             for num in range(0, int(extra_specs['hw:numa_nodes'])):
 407                 numa_cpu_node = "hw:numa_cpus.%s" % num
 408                 numa_mem_node = "hw:numa_mem.%s" % num
 409                 numacpu_key = "numaCpu-%s" % num
 410                 numamem_key = "numaMem-%s" % num
 411
 412                 if (extra_specs.get(numa_cpu_node) and
 413                         extra_specs.get(numa_mem_node)):
 414                     numa_capability['hpa-feature-attributes'].append({
 415                         'hpa-attribute-key': numacpu_key,
 416                         'hpa-attribute-value': json.dumps({'value': str(
 417                             extra_specs[numa_cpu_node])})
 418                         })
 419                     numa_capability['hpa-feature-attributes'].append({
 420                         'hpa-attribute-key': numamem_key,
 421                         'hpa-attribute-value': json.dumps({'value': str(
 422                             extra_specs[numa_mem_node]), 'unit': 'MB'})
 423                         })
 424
 425         return numa_capability
 426
 427     def _get_storage_capabilities(self, flavor):
 428         storage_capability = {}
 429         feature_uuid = uuid.uuid4()
 430
 431         storage_capability['hpa-capability-id'] = str(feature_uuid)
 432         storage_capability['hpa-feature'] = 'localStorage'
 433         storage_capability['architecture'] = 'generic'
 434         storage_capability['hpa-version'] = 'v1'
 435
 436         storage_capability['hpa-feature-attributes'] = []
 437         storage_capability['hpa-feature-attributes'].append({
 438             'hpa-attribute-key': 'diskSize',
 439             'hpa-attribute-value': json.dumps({'value': str(
 440                 flavor['disk']), 'unit': 'MB'})
 441             })
 442         storage_capability['hpa-feature-attributes'].append({
 443             'hpa-attribute-key': 'swapMemSize',
 444             'hpa-attribute-value': json.dumps({'value': str(
 445                 flavor.get('swap', 0)), 'unit': 'MB'})
 446             })
 447         storage_capability['hpa-feature-attributes'].append({
 448             'hpa-attribute-key': 'ephemeralDiskSize',
 449             'hpa-attribute-value': json.dumps({'value': str(
 450                 flavor.get('OS-FLV-EXT-DATA:ephemeral', 0)), 'unit': 'GB'})
 451             })
 452         return storage_capability
 453
 454     def _get_instruction_set_capabilities(self, extra_specs):
 455         instruction_capability = {}
 456         feature_uuid = uuid.uuid4()
 457
 458         if extra_specs.get('hw:capabilities:cpu_info:features'):
 459             instruction_capability['hpa-capability-id'] = str(feature_uuid)
 460             instruction_capability['hpa-feature'] = 'instructionSetExtensions'
 461             instruction_capability['architecture'] = 'Intel64'
 462             instruction_capability['hpa-version'] = 'v1'
 463
 464             instruction_capability['hpa-feature-attributes'] = []
 465             instruction_capability['hpa-feature-attributes'].append({
 466                 'hpa-attribute-key': 'instructionSetExtensions',
 467                 'hpa-attribute-value': json.dumps(
 468                     {'value': extra_specs[
 469                         'hw:capabilities:cpu_info:features']})
 470                 })
 471         return instruction_capability
 472
 473     def _get_pci_passthrough_capabilities(self, extra_specs):
 474         instruction_capability = {}
 475         feature_uuid = uuid.uuid4()
 476
 477         if extra_specs.get('pci_passthrough:alias'):
 478             value1 = extra_specs['pci_passthrough:alias'].split(':')
 479             value2 = value1[0].split('-')
 480
 481             instruction_capability['hpa-capability-id'] = str(feature_uuid)
 482             instruction_capability['hpa-feature'] = 'pciePassthrough'
 483             instruction_capability['architecture'] = str(value2[2])
 484             instruction_capability['hpa-version'] = 'v1'
 485
 486             instruction_capability['hpa-feature-attributes'] = []
 487             instruction_capability['hpa-feature-attributes'].append({
 488                 'hpa-attribute-key': 'pciCount',
 489                 'hpa-attribute-value': json.dumps({'value': value1[1]})
 490                 })
 491             instruction_capability['hpa-feature-attributes'].append({
 492                 'hpa-attribute-key': 'pciVendorId',
 493                 'hpa-attribute-value': json.dumps({'value': value2[3]})
 494                 })
 495             instruction_capability['hpa-feature-attributes'].append({
 496                 'hpa-attribute-key': 'pciDeviceId',
 497                 'hpa-attribute-value': json.dumps({'value': value2[4]})
 498                 })
 499
 500         return instruction_capability
 501
 502     def _get_ovsdpdk_capabilities(self):
 503         ovsdpdk_capability = {}
 504         feature_uuid = uuid.uuid4()
 505
 506         if not self._vim_info:
 507             self._vim_info = self.get_vim(get_all=True)
 508         cloud_extra_info_str = self._vim_info.get('cloud-extra-info')
 509         if not isinstance(cloud_extra_info_str, dict):
 510             try:
 511                 cloud_extra_info_str = json.loads(cloud_extra_info_str)
 512             except Exception as ex:
 513                 logger.error("Can not convert cloud extra info %s %s" % (
 514                              str(ex), cloud_extra_info_str))
 515                 return {}
 516         if cloud_extra_info_str:
 517             cloud_dpdk_info = cloud_extra_info_str.get("ovsDpdk")
 518             if cloud_dpdk_info:
 519                 ovsdpdk_capability['hpa-capability-id'] = str(feature_uuid)
 520                 ovsdpdk_capability['hpa-feature'] = 'ovsDpdk'
 521                 ovsdpdk_capability['architecture'] = 'Intel64'
 522                 ovsdpdk_capability['hpa-version'] = 'v1'
 523
 524                 ovsdpdk_capability['hpa-feature-attributes'] = []
 525                 ovsdpdk_capability['hpa-feature-attributes'].append({
 526                     'hpa-attribute-key': str(cloud_dpdk_info.get("libname")),
 527                     'hpa-attribute-value': json.dumps(
 528                         {'value': cloud_dpdk_info.get("libversion")})
 529                     })
 530         return ovsdpdk_capability