test/vcpe/vcpecommon.py

   1 #!/usr/bin/env python
   2
   3 import json
   4 import logging
   5 import os
   6 import pickle
   7 import re
   8 import sys
   9
  10 import ipaddress
  11 import mysql.connector
  12 import requests
  13 import subprocess
  14 import time
  15 import yaml
  16 from novaclient import client as openstackclient
  17 from openstack.config import loader
  18 from kubernetes import client, config
  19 from netaddr import IPAddress, IPNetwork
  20
  21 class VcpeCommon:
  22
  23     def __init__(self, extra_host_names=None, cfg_file=None):
  24         self.logger = logging.getLogger(__name__)
  25         self.logger.setLevel(logging.DEBUG)
  26         self.logger.info('Initializing configuration')
  27         self.default_config = 'vcpeconfig.yaml'
  28
  29         # Read configuration from config file
  30         self._load_config(cfg_file)
  31         # Load OpenStack settings
  32         self._load_os_config()
  33
  34         self.sdnc_controller_pod = '-'.join([self.onap_environment, 'sdnc-sdnc-0'])
  35         # OOM: this is the address that the brg and bng will nat for sdnc access - 10.0.0.x address of k8 host for sdnc-0 container
  36         self.sdnc_oam_ip = self.get_pod_node_oam_ip(self.sdnc_controller_pod)
  37         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
  38         self.oom_so_sdnc_aai_ip = self.get_pod_node_public_ip(self.sdnc_controller_pod)
  39         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
  40         self.oom_dcae_ves_collector = self.oom_so_sdnc_aai_ip
  41         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
  42         self.mr_ip_addr = self.oom_so_sdnc_aai_ip
  43         self.mr_ip_port = '30227'
  44         self.so_nbi_port = '30277' if self.oom_mode else '8080'
  45         self.sdnc_preloading_port = '30267' if self.oom_mode else '8282'
  46         self.aai_query_port = '30233' if self.oom_mode else '8443'
  47         self.sniro_port = '30288' if self.oom_mode else '8080'
  48
  49         self.host_names = ['sdc', 'so', 'sdnc', 'robot', 'aai-inst1', self.dcae_ves_collector_name, 'mariadb-galera']
  50         if extra_host_names:
  51             self.host_names.extend(extra_host_names)
  52         # get IP addresses
  53         self.hosts = self.get_vm_ip(self.host_names, self.external_net_addr, self.external_net_prefix_len)
  54         # this is the keyword used to name vgw stack, must not be used in other stacks
  55         self.vgw_name_keyword = 'base_vcpe_vgw'
  56         # this is the file that will keep the index of last assigned SO name
  57         self.vgw_vfmod_name_index_file= '__var/vgw_vfmod_name_index'
  58         self.svc_instance_uuid_file = '__var/svc_instance_uuid'
  59         self.preload_dict_file = '__var/preload_dict'
  60         self.vgmux_vnf_name_file = '__var/vgmux_vnf_name'
  61         self.product_family_id = 'f9457e8c-4afd-45da-9389-46acd9bf5116'
  62         self.custom_product_family_id = 'a9a77d5a-123e-4ca2-9eb9-0b015d2ee0fb'
  63         self.instance_name_prefix = {
  64             'service': 'vcpe_svc',
  65             'network': 'vcpe_net',
  66             'vnf': 'vcpe_vnf',
  67             'vfmodule': 'vcpe_vfmodule'
  68         }
  69         self.aai_userpass = 'AAI', 'AAI'
  70         self.os_tenant_id = self.cloud['--os-tenant-id']
  71         self.os_region_name = self.cloud['--os-region-name']
  72         self.common_preload_config['pub_key'] = self.pub_key
  73         self.sniro_url = 'http://' + self.hosts['robot'] + ':' + self.sniro_port + '/__admin/mappings'
  74         self.sniro_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
  75         self.homing_solution = 'sniro'  # value is either 'sniro' or 'oof'
  76 #        self.homing_solution = 'oof'
  77         self.customer_location_used_by_oof = {
  78             "customerLatitude": "32.897480",
  79             "customerLongitude": "-97.040443",
  80             "customerName": "some_company"
  81         }
  82
  83         #############################################################################################
  84         # SDC urls
  85         self.sdc_be_port = '30204'
  86         self.sdc_be_request_userpass = 'vid', 'Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U'
  87         self.sdc_be_request_headers = {'X-ECOMP-InstanceID': 'VID'}
  88         self.sdc_be_url_prefix = 'https://' + self.hosts['sdc'] + ':' + self.sdc_be_port
  89         self.sdc_service_list_url = self.sdc_be_url_prefix + '/sdc/v1/catalog/services'
  90
  91         self.sdc_fe_port = '30207'
  92         self.sdc_fe_request_userpass = 'beep', 'boop'
  93         self.sdc_fe_request_headers = {'USER_ID': 'demo', 'Content-Type': 'application/json'}
  94         self.sdc_fe_url_prefix = 'https://' + self.hosts['sdc'] + ':' + self.sdc_fe_port
  95         self.sdc_get_category_list_url = self.sdc_fe_url_prefix + '/sdc1/feProxy/rest/v1/categories'
  96         self.sdc_create_allotted_resource_subcategory_url = self.sdc_fe_url_prefix + '/sdc1/feProxy/rest/v1/category/resources/resourceNewCategory.allotted%20resource/subCategory'
  97
  98         #############################################################################################
  99         # SDNC urls
 100         self.sdnc_userpass = 'admin', 'Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U'
 101         self.sdnc_db_name = 'sdnctl'
 102         self.sdnc_db_user = 'sdnctl'
 103         self.sdnc_db_pass = 'gamma'
 104         self.sdnc_db_port = self.get_k8s_service_endpoint_info('mariadb-galera','port') if self.oom_mode else '3306'
 105         self.sdnc_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 106         self.sdnc_preload_network_url = 'https://' + self.hosts['sdnc'] + \
 107                                         ':' + self.sdnc_preloading_port + '/restconf/operations/VNF-API:preload-network-topology-operation'
 108         self.sdnc_preload_network_gra_url = 'https://' + self.hosts['sdnc'] + \
 109                                         ':' + self.sdnc_preloading_port + '/restconf/operations/GENERIC-RESOURCE-API:preload-network-topology-operation'
 110         self.sdnc_preload_vnf_url = 'https://' + self.hosts['sdnc'] + \
 111                                     ':' + self.sdnc_preloading_port + '/restconf/operations/VNF-API:preload-vnf-topology-operation'
 112         self.sdnc_preload_gra_url = 'https://' + self.hosts['sdnc'] + \
 113                                     ':' + self.sdnc_preloading_port + '/restconf/operations/GENERIC-RESOURCE-API:preload-vf-module-topology-operation'
 114         self.sdnc_ar_cleanup_url = 'https://' + self.hosts['sdnc'] + ':' + self.sdnc_preloading_port + \
 115                                    '/restconf/config/GENERIC-RESOURCE-API:'
 116
 117         #############################################################################################
 118         # MARIADB-GALERA settings
 119         self.mariadb_galera_endpoint_ip = self.get_k8s_service_endpoint_info('mariadb-galera','ip')
 120         self.mariadb_galera_endpoint_port = self.get_k8s_service_endpoint_info('mariadb-galera','port')
 121
 122         #############################################################################################
 123         # SO urls, note: do NOT add a '/' at the end of the url
 124         self.so_req_api_url = {'v4': 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/serviceInstantiation/v7/serviceInstances',
 125                            'v5': 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/serviceInstantiation/v7/serviceInstances'}
 126         self.so_check_progress_api_url = 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/orchestrationRequests/v6'
 127         self.so_userpass = 'InfraPortalClient', 'password1$'
 128         self.so_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 129         self.so_db_name = 'catalogdb'
 130         self.so_db_user = 'root'
 131         self.so_db_pass = 'secretpassword'
 132         self.so_db_host = self.mariadb_galera_endpoint_ip if self.oom_mode else self.hosts['so']
 133         self.so_db_port = self.mariadb_galera_endpoint_port if self.oom_mode else '3306'
 134
 135         self.vpp_inf_url = 'http://{0}:8183/restconf/config/ietf-interfaces:interfaces'
 136         self.vpp_api_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 137         self.vpp_api_userpass = ('admin', 'admin')
 138         self.vpp_ves_url= 'http://{0}:8183/restconf/config/vesagent:vesagent'
 139
 140         #############################################################################################
 141         # POLICY urls
 142         self.policy_userpass = ('healthcheck', 'zb!XztG34')
 143         self.policy_headers = {'Accept': 'application/json', 'Content-Type': 'application/json'}
 144         self.policy_api_url = 'https://{0}:6969/policy/api/v1/policytypes/onap.policies.controlloop.Operational/versions/1.0.0/policies'
 145         self.policy_pap_get_url = 'https://{0}:6969/policy/pap/v1/pdps'
 146         self.policy_pap_json = {'policies': [{'policy-id': 'operational.vcpe'}]}
 147         self.policy_pap_post_url = self.policy_pap_get_url + '/policies'
 148         self.policy_api_service_name = 'policy-api'
 149         self.policy_pap_service_name = 'policy-pap'
 150
 151         #############################################################################################
 152         # AAI urls
 153         self.aai_region_query_url = 'https://' + self.oom_so_sdnc_aai_ip + ':' +\
 154                                     self.aai_query_port +\
 155                                     '/aai/v14/cloud-infrastructure/cloud-regions/cloud-region/CloudOwner/' +\
 156                                     self.cloud['--os-region-name']
 157         self.aai_headers = {'Accept': 'application/json',
 158                             'Content-Type': 'application/json',
 159                             'X-FromAppId': 'postman', 'X-TransactionId': '9999'}
 160
 161     def _load_config(self, cfg_file):
 162         """
 163         Reads vcpe config file and injects settings as object's attributes
 164         :param cfg_file: Configuration file path
 165         """
 166
 167         if cfg_file is None:
 168             cfg_file = self.default_config
 169
 170         try:
 171             with open(cfg_file, 'r') as cfg:
 172                 cfg_yml = yaml.full_load(cfg)
 173         except Exception as e:
 174             self.logger.error('Error loading configuration: ' + str(e))
 175             sys.exit(1)
 176
 177         self.logger.debug('\n' + yaml.dump(cfg_yml))
 178
 179         # Use setattr to load config file keys as VcpeCommon class' object
 180         # attributes
 181         try:
 182             # Check config isn't empty
 183             if cfg_yml is not None:
 184                 for cfg_key in cfg_yml:
 185                     setattr(self, cfg_key, cfg_yml[cfg_key])
 186         except TypeError as e:
 187             self.logger.error('Unable to parse config file: ' + str(e))
 188             sys.exit(1)
 189
 190     def _load_os_config(self):
 191         """
 192         Reads cloud settings and sets them as object's 'cloud' attribute
 193         """
 194         # Create OpenStackConfig config instance
 195         os_config = loader.OpenStackConfig()
 196         # Try reading cloud settings for self.cloud_name
 197         try:
 198             os_cloud = os_config.cloud_config['clouds'][self.cloud_name]
 199         except KeyError:
 200             self.logger.error('Error fetching cloud settings for cloud "{0}"'
 201                               .format(self.cloud_name))
 202             sys.exit(1)
 203         self.logger.debug('Cloud config:\n {0}'.format(json.dumps(
 204                           os_cloud,indent=4)))
 205
 206         # Extract all OS settings keys and alter their names
 207         # to conform to openstack cli client
 208         self.cloud = {}
 209         for k in os_cloud:
 210             if isinstance(os_cloud[k],dict):
 211                 for sub_k in os_cloud[k]:
 212                     os_setting_name = '--os-' + sub_k.replace('_','-')
 213                     self.cloud[os_setting_name] = os_cloud[k][sub_k]
 214             else:
 215                 os_setting_name = '--os-' + k.replace('_','-')
 216                 self.cloud[os_setting_name] = os_cloud[k]
 217
 218     def heatbridge(self, openstack_stack_name, svc_instance_uuid):
 219         """
 220         Add vserver information to AAI
 221         """
 222         self.logger.info('Adding vServer information to AAI for {0}'.format(openstack_stack_name))
 223         if not self.oom_mode:
 224             cmd = '/opt/demo.sh heatbridge {0} {1} vCPE'.format(openstack_stack_name, svc_instance_uuid)
 225             ret = subprocess.getstatusoutput("ssh -i onap_dev root@{0} '{1}'".format(self.hosts['robot'], cmd))
 226             self.logger.debug('%s', ret)
 227         else:
 228             print('To add vGMUX vserver info to AAI, do the following:')
 229             print('- ssh to rancher')
 230             print('- sudo su -')
 231             print('- cd /root/oom/kubernetes/robot')
 232             print('- ./demo-k8s.sh onap heatbridge {0} {1} vCPE'.format(openstack_stack_name, svc_instance_uuid))
 233
 234     def get_brg_mac_from_sdnc(self):
 235         """
 236         Check table DHCP_MAP in the SDNC DB. Find the newly instantiated BRG MAC address.
 237         Note that there might be multiple BRGs, the most recently instantiated BRG always has the largest IP address.
 238         """
 239         if self.oom_mode:
 240             db_host=self.mariadb_galera_endpoint_ip
 241         else:
 242             db_host=self.hosts['mariadb-galera']
 243
 244         cnx = mysql.connector.connect(user=self.sdnc_db_user,
 245                                       password=self.sdnc_db_pass,
 246                                       database=self.sdnc_db_name,
 247                                       host=db_host,
 248                                       port=self.sdnc_db_port)
 249         cursor = cnx.cursor()
 250         query = "SELECT * from DHCP_MAP"
 251         cursor.execute(query)
 252
 253         self.logger.debug('DHCP_MAP table in SDNC')
 254         mac_recent = None
 255         host = -1
 256         for mac, ip in cursor:
 257             self.logger.debug(mac + ' - ' + ip)
 258             this_host = int(ip.split('.')[-1])
 259             if host < this_host:
 260                 host = this_host
 261                 mac_recent = mac
 262
 263         cnx.close()
 264
 265         try:
 266             assert mac_recent
 267         except AssertionError:
 268             self.logger.error('Failed to obtain BRG MAC address from database')
 269             sys.exit(1)
 270
 271         return mac_recent
 272
 273     def execute_cmds_mariadb(self, cmds):
 274         self.execute_cmds_db(cmds, self.sdnc_db_user, self.sdnc_db_pass,
 275                              self.sdnc_db_name, self.mariadb_galera_endpoint_ip,
 276                              self.mariadb_galera_endpoint_port)
 277
 278     def execute_cmds_sdnc_db(self, cmds):
 279         self.execute_cmds_db(cmds, self.sdnc_db_user, self.sdnc_db_pass, self.sdnc_db_name,
 280                              self.hosts['sdnc'], self.sdnc_db_port)
 281
 282     def execute_cmds_so_db(self, cmds):
 283         self.execute_cmds_db(cmds, self.so_db_user, self.so_db_pass, self.so_db_name,
 284                              self.so_db_host, self.so_db_port)
 285
 286     def execute_cmds_db(self, cmds, dbuser, dbpass, dbname, host, port):
 287         cnx = mysql.connector.connect(user=dbuser, password=dbpass, database=dbname, host=host, port=port)
 288         cursor = cnx.cursor()
 289         for cmd in cmds:
 290             self.logger.debug(cmd)
 291             cursor.execute(cmd)
 292             self.logger.debug('%s', cursor)
 293         cnx.commit()
 294         cursor.close()
 295         cnx.close()
 296
 297     def find_file(self, file_name_keyword, file_ext, search_dir):
 298         """
 299         :param file_name_keyword:  keyword used to look for the csar file, case insensitive matching, e.g, infra
 300         :param file_ext: e.g., csar, json
 301         :param search_dir path to search
 302         :return: path name of the file
 303         """
 304         file_name_keyword = file_name_keyword.lower()
 305         file_ext = file_ext.lower()
 306         if not file_ext.startswith('.'):
 307             file_ext = '.' + file_ext
 308
 309         filenamepath = None
 310         for file_name in os.listdir(search_dir):
 311             file_name_lower = file_name.lower()
 312             if file_name_keyword in file_name_lower and file_name_lower.endswith(file_ext):
 313                 if filenamepath:
 314                     self.logger.error('Multiple files found for *{0}*.{1} in '
 315                                       'directory {2}'.format(file_name_keyword, file_ext, search_dir))
 316                     sys.exit(1)
 317                 filenamepath = os.path.abspath(os.path.join(search_dir, file_name))
 318
 319         if filenamepath:
 320             return filenamepath
 321         else:
 322             self.logger.error("Cannot find *{0}*{1} in directory {2}".format(file_name_keyword, file_ext, search_dir))
 323             sys.exit(1)
 324
 325     @staticmethod
 326     def network_name_to_subnet_name(network_name):
 327         """
 328         :param network_name: example: vcpe_net_cpe_signal_201711281221
 329         :return: vcpe_net_cpe_signal_subnet_201711281221
 330         """
 331         fields = network_name.split('_')
 332         fields.insert(-1, 'subnet')
 333         return '_'.join(fields)
 334
 335     def set_network_name(self, network_name):
 336         param = ' '.join([k + ' ' + v for k, v in self.cloud.items()])
 337         openstackcmd = 'openstack ' + param
 338         cmd = ' '.join([openstackcmd, 'network set --name', network_name, 'ONAP-NW1'])
 339         os.popen(cmd)
 340
 341     def set_subnet_name(self, network_name):
 342         """
 343         Example: network_name =  vcpe_net_cpe_signal_201711281221
 344         set subnet name to vcpe_net_cpe_signal_subnet_201711281221
 345         :return:
 346         """
 347         param = ' '.join([k + ' ' + v for k, v in self.cloud.items()])
 348         openstackcmd = 'openstack ' + param
 349
 350         # expected results: | subnets | subnet_id |
 351         subnet_info = os.popen(openstackcmd + ' network show ' + network_name + ' |grep subnets').read().split('|')
 352         if len(subnet_info) > 2 and subnet_info[1].strip() == 'subnets':
 353             subnet_id = subnet_info[2].strip()
 354             subnet_name = self.network_name_to_subnet_name(network_name)
 355             cmd = ' '.join([openstackcmd, 'subnet set --name', subnet_name, subnet_id])
 356             os.popen(cmd)
 357             self.logger.info("Subnet name set to: " + subnet_name)
 358             return True
 359         else:
 360             self.logger.error("Can't get subnet info from network name: " + network_name)
 361             return False
 362
 363     def set_closed_loop_policy(self, policy_template_file):
 364         # Gather policy services cluster ips
 365         p_api_cluster_ip = self.get_k8s_service_cluster_ip(self.policy_api_service_name)
 366         p_pap_cluster_ip = self.get_k8s_service_cluster_ip(self.policy_pap_service_name)
 367
 368         # Read policy json from file
 369         with open(policy_template_file) as f:
 370             try:
 371                 policy_json = json.load(f)
 372             except ValueError:
 373                 self.logger.error(policy_template_file + " doesn't seem to contain valid JSON data")
 374                 sys.exit(1)
 375
 376         # Check policy already applied
 377         policy_exists_req = requests.get(self.policy_pap_get_url.format(
 378                             p_pap_cluster_ip), auth=self.policy_userpass,
 379                             verify=False, headers=self.policy_headers)
 380         if policy_exists_req.status_code != 200:
 381             self.logger.error('Failure in checking CL policy existence. '
 382                                'Policy-pap responded with HTTP code {0}'.format(
 383                                policy_exists_req.status_code))
 384             sys.exit(1)
 385
 386         try:
 387             policy_exists_json = policy_exists_req.json()
 388         except ValueError as e:
 389             self.logger.error('Policy-pap request failed: ' + e.message)
 390             sys.exit(1)
 391
 392         try:
 393             assert policy_exists_json['groups'][0]['pdpSubgroups'] \
 394                                [1]['policies'][0]['name'] != 'operational.vcpe'
 395         except AssertionError:
 396             self.logger.info('vCPE closed loop policy already exists, not applying')
 397             return
 398         except IndexError:
 399             pass # policy doesn't exist
 400
 401         # Create policy
 402         policy_create_req = requests.post(self.policy_api_url.format(
 403                             p_api_cluster_ip), auth=self.policy_userpass,
 404                             json=policy_json, verify=False,
 405                             headers=self.policy_headers)
 406         # Get the policy id from policy-api response
 407         if policy_create_req.status_code != 200:
 408             self.logger.error('Failed creating policy. Policy-api responded'
 409                               ' with HTTP code {0}'.format(policy_create_req.status_code))
 410             sys.exit(1)
 411
 412         try:
 413             policy_version = json.loads(policy_create_req.text)['policy-version']
 414         except (KeyError, ValueError):
 415             self.logger.error('Policy API response not understood:')
 416             self.logger.debug('\n' + str(policy_create_req.text))
 417
 418         # Inject the policy into Policy PAP
 419         self.policy_pap_json['policies'].append({'policy-version': policy_version})
 420         policy_insert_req = requests.post(self.policy_pap_post_url.format(
 421                             p_pap_cluster_ip), auth=self.policy_userpass,
 422                             json=self.policy_pap_json, verify=False,
 423                             headers=self.policy_headers)
 424         if policy_insert_req.status_code != 200:
 425             self.logger.error('Policy PAP request failed with HTTP code'
 426                               '{0}'.format(policy_insert_req.status_code))
 427             sys.exit(1)
 428         self.logger.info('Successully pushed closed loop Policy')
 429
 430     def is_node_in_aai(self, node_type, node_uuid):
 431         key = None
 432         search_node_type = None
 433         if node_type == 'service':
 434             search_node_type = 'service-instance'
 435             key = 'service-instance-id'
 436         elif node_type == 'vnf':
 437             search_node_type = 'generic-vnf'
 438             key = 'vnf-id'
 439         else:
 440             logging.error('Invalid node_type: ' + node_type)
 441             sys.exit(1)
 442
 443         url = 'https://{0}:{1}/aai/v11/search/nodes-query?search-node-type={2}&filter={3}:EQUALS:{4}'.format(
 444             self.hosts['aai-inst1'], self.aai_query_port, search_node_type, key, node_uuid)
 445
 446         headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'X-FromAppID': 'vCPE-Robot', 'X-TransactionId': 'get_aai_subscr'}
 447         r = requests.get(url, headers=headers, auth=self.aai_userpass, verify=False)
 448         response = r.json()
 449         self.logger.debug('aai query: ' + url)
 450         self.logger.debug('aai response:\n' + json.dumps(response, indent=4, sort_keys=True))
 451         return 'result-data' in response
 452
 453     @staticmethod
 454     def extract_ip_from_str(net_addr, net_addr_len, sz):
 455         """
 456         :param net_addr:  e.g. 10.5.12.0
 457         :param net_addr_len: e.g. 24
 458         :param sz: a string
 459         :return: the first IP address matching the network, e.g. 10.5.12.3
 460         """
 461         network = ipaddress.ip_network(unicode('{0}/{1}'.format(net_addr, net_addr_len)), strict=False) # pylint: disable=E0602
 462         ip_list = re.findall(r'[0-9]+(?:\.[0-9]+){3}', sz)
 463         for ip in ip_list:
 464             this_net = ipaddress.ip_network(unicode('{0}/{1}'.format(ip, net_addr_len)), strict=False) # pylint: disable=E0602
 465             if this_net == network:
 466                 return str(ip)
 467         return None
 468
 469     def get_pod_node_oam_ip(self, pod):
 470         """
 471         :Assuming kubectl is available and configured by default config (~/.kube/config)
 472         :param pod: pod name substring, e.g. 'sdnc-sdnc-0'
 473         :return pod's cluster node oam ip (10.0.0.0/16)
 474         """
 475         ret = None
 476         config.load_kube_config()
 477         api = client.CoreV1Api()
 478         kslogger = logging.getLogger('kubernetes')
 479         kslogger.setLevel(logging.INFO)
 480         res = api.list_pod_for_all_namespaces()
 481         for i in res.items:
 482             if pod in i.metadata.name:
 483                 self.logger.debug("found {0}\t{1}\t{2}".format(i.metadata.name, i.status.host_ip, i.spec.node_name))
 484                 ret = i.status.host_ip
 485                 break
 486
 487         if ret is None:
 488             ret = raw_input("Enter " + self.sdnc_controller_pod + " pod cluster node OAM IP address(10.0.0.0/16): ") # pylint: disable=E0602
 489         return ret
 490
 491     def get_pod_node_public_ip(self, pod):
 492         """
 493         :Assuming kubectl is available and configured by default config (~/.kube/config)
 494         :param pod: pod name substring, e.g. 'sdnc-sdnc-0'
 495         :return pod's cluster node public ip (i.e. 10.12.0.0/16)
 496         """
 497         ret = None
 498         config.load_kube_config()
 499         api = client.CoreV1Api()
 500         kslogger = logging.getLogger('kubernetes')
 501         kslogger.setLevel(logging.INFO)
 502         res = api.list_pod_for_all_namespaces()
 503         for i in res.items:
 504             if pod in i.metadata.name:
 505                 ret = self.get_vm_public_ip_by_nova(i.spec.node_name)
 506                 self.logger.debug("found node {0} public ip: {1}".format(i.spec.node_name, ret))
 507                 break
 508
 509         if ret is None:
 510             ret = raw_input("Enter " + self.sdnc_controller_pod + " pod cluster node public IP address(i.e. " + self.external_net_addr + "): ") # pylint: disable=E0602
 511         return ret
 512
 513     def get_vm_public_ip_by_nova(self, vm):
 514         """
 515         This method uses openstack nova api to retrieve vm public ip
 516         :param vm: vm name
 517         :return vm public ip
 518         """
 519         subnet = IPNetwork('{0}/{1}'.format(self.external_net_addr, self.external_net_prefix_len))
 520         nova = openstackclient.Client(2, self.cloud['--os-username'], self.cloud['--os-password'], self.cloud['--os-tenant-id'], self.cloud['--os-auth-url'])
 521         for i in nova.servers.list():
 522             if i.name == vm:
 523                 for k, v in i.networks.items(): # pylint: disable=W0612
 524                     for ip in v:
 525                         if IPAddress(ip) in subnet:
 526                             return ip
 527         return None
 528
 529     def get_vm_ip(self, keywords, net_addr=None, net_addr_len=None):
 530         """
 531         :param keywords: list of keywords to search for vm, e.g. ['bng', 'gmux', 'brg']
 532         :param net_addr: e.g. 10.12.5.0
 533         :param net_addr_len: e.g. 24
 534         :return: dictionary {keyword: ip}
 535         """
 536         if not net_addr:
 537             net_addr = self.external_net_addr
 538
 539         if not net_addr_len:
 540             net_addr_len = self.external_net_prefix_len
 541
 542         param = ' '.join([k + ' ' + v for k, v in self.cloud.items() if 'identity' not in k])
 543         openstackcmd = 'nova ' + param + ' list'
 544         self.logger.debug(openstackcmd)
 545
 546         results = os.popen(openstackcmd).read()
 547         all_vm_ip_dict = self.extract_vm_ip_as_dict(results, net_addr, net_addr_len)
 548         latest_vm_list = self.remove_old_vms(all_vm_ip_dict.keys(), self.cpe_vm_prefix)
 549         latest_vm_ip_dict = {vm: all_vm_ip_dict[vm] for vm in latest_vm_list}
 550         ip_dict = self.select_subset_vm_ip(latest_vm_ip_dict, keywords)
 551         if self.oom_mode:
 552             ip_dict.update(self.get_oom_onap_vm_ip(keywords))
 553
 554         if len(ip_dict) != len(keywords):
 555             self.logger.error('Cannot find all desired IP addresses for %s.', keywords)
 556             self.logger.error(json.dumps(ip_dict, indent=4, sort_keys=True))
 557             self.logger.error('Temporarily continue.. remember to check back vcpecommon.py line: 396')
 558 #            sys.exit(1)
 559         return ip_dict
 560
 561     def get_oom_onap_vm_ip(self, keywords):
 562         vm_ip = {}
 563         for vm in keywords:
 564             if vm in self.host_names:
 565                 vm_ip[vm] = self.oom_so_sdnc_aai_ip
 566         return vm_ip
 567
 568     def get_k8s_service_cluster_ip(self, service):
 569         """
 570         Returns cluster IP for a given service
 571         :param service: name of the service
 572         :return: cluster ip
 573         """
 574         config.load_kube_config()
 575         api = client.CoreV1Api()
 576         kslogger = logging.getLogger('kubernetes')
 577         kslogger.setLevel(logging.INFO)
 578         try:
 579             resp = api.read_namespaced_service(service, self.onap_namespace)
 580         except client.rest.ApiException as e:
 581             self.logger.error('Error while making k8s API request: ' + e.body)
 582             sys.exit(1)
 583
 584         return resp.spec.cluster_ip
 585
 586     def get_k8s_service_endpoint_info(self, service, subset):
 587         """
 588         Returns endpoint data for a given service and subset. If there
 589         is more than one endpoint returns data for the first one from
 590         the list that API returned.
 591         :param service: name of the service
 592         :param subset: subset name, one of "ip","port"
 593         :return: endpoint ip
 594         """
 595         config.load_kube_config()
 596         api = client.CoreV1Api()
 597         kslogger = logging.getLogger('kubernetes')
 598         kslogger.setLevel(logging.INFO)
 599         try:
 600             resp = api.read_namespaced_endpoints(service, self.onap_namespace)
 601         except client.rest.ApiException as e:
 602             self.logger.error('Error while making k8s API request: ' + e.body)
 603             sys.exit(1)
 604
 605         if subset == "ip":
 606             return resp.subsets[0].addresses[0].ip
 607         elif subset == "port":
 608             return resp.subsets[0].ports[0].port
 609         else:
 610             self.logger.error("Unsupported subset type")
 611
 612     def extract_vm_ip_as_dict(self, novalist_results, net_addr, net_addr_len):
 613         vm_ip_dict = {}
 614         for line in novalist_results.split('\n'):
 615             fields = line.split('|')
 616             if len(fields) == 8:
 617                 vm_name = fields[2]
 618                 ip_info = fields[-2]
 619                 ip = self.extract_ip_from_str(net_addr, net_addr_len, ip_info)
 620                 vm_ip_dict[vm_name] = ip
 621
 622         return vm_ip_dict
 623
 624     def remove_old_vms(self, vm_list, prefix):
 625         """
 626         For vms with format name_timestamp, only keep the one with the latest timestamp.
 627         E.g.,
 628             zdcpe1cpe01brgemu01_201805222148        (drop this)
 629             zdcpe1cpe01brgemu01_201805222229        (keep this)
 630             zdcpe1cpe01gw01_201805162201
 631         """
 632         new_vm_list = []
 633         same_type_vm_dict = {}
 634         for vm in vm_list:
 635             fields = vm.split('_')
 636             if vm.startswith(prefix) and len(fields) == 2 and len(fields[-1]) == len('201805222148') and fields[-1].isdigit():
 637                 if vm > same_type_vm_dict.get(fields[0], '0'):
 638                     same_type_vm_dict[fields[0]] = vm
 639             else:
 640                 new_vm_list.append(vm)
 641
 642         new_vm_list.extend(same_type_vm_dict.values())
 643         return new_vm_list
 644
 645     def select_subset_vm_ip(self, all_vm_ip_dict, vm_name_keyword_list):
 646         vm_ip_dict = {}
 647         for keyword in vm_name_keyword_list:
 648             for vm, ip in all_vm_ip_dict.items():
 649                 if keyword in vm:
 650                     vm_ip_dict[keyword] = ip
 651                     break
 652         return vm_ip_dict
 653
 654     def del_vgmux_ves_mode(self):
 655         url = self.vpp_ves_url.format(self.hosts['mux']) + '/mode'
 656         r = requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 657         self.logger.debug('%s', r)
 658
 659     def del_vgmux_ves_collector(self):
 660         url = self.vpp_ves_url.format(self.hosts['mux']) + '/config'
 661         r = requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 662         self.logger.debug('%s', r)
 663
 664     def set_vgmux_ves_collector(self ):
 665         url = self.vpp_ves_url.format(self.hosts['mux'])
 666         data = {'config':
 667                     {'server-addr': self.hosts[self.dcae_ves_collector_name],
 668                      'server-port': '30235' if self.oom_mode else '8081',
 669                      'read-interval': '10',
 670                      'is-add':'1'
 671                      }
 672                 }
 673         r = requests.post(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass, json=data)
 674         self.logger.debug('%s', r)
 675
 676     def set_vgmux_packet_loss_rate(self, lossrate, vg_vnf_instance_name):
 677         url = self.vpp_ves_url.format(self.hosts['mux'])
 678         data = {"mode":
 679                     {"working-mode": "demo",
 680                      "base-packet-loss": str(lossrate),
 681                      "source-name": vg_vnf_instance_name
 682                      }
 683                 }
 684         r = requests.post(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass, json=data)
 685         self.logger.debug('%s', r)
 686
 687         # return all the VxLAN interface names of BRG or vGMUX based on the IP address
 688     def get_vxlan_interfaces(self, ip, print_info=False):
 689         url = self.vpp_inf_url.format(ip)
 690         self.logger.debug('url is this: %s', url)
 691         r = requests.get(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 692         data = r.json()['interfaces']['interface']
 693         if print_info:
 694             for inf in data:
 695                 if 'name' in inf and 'type' in inf and inf['type'] == 'v3po:vxlan-tunnel':
 696                     print(json.dumps(inf, indent=4, sort_keys=True))
 697
 698         return [inf['name'] for inf in data if 'name' in inf and 'type' in inf and inf['type'] == 'v3po:vxlan-tunnel']
 699
 700     # delete all VxLAN interfaces of each hosts
 701     def delete_vxlan_interfaces(self, host_dic):
 702         for host, ip in host_dic.items():
 703             deleted = False
 704             self.logger.info('{0}: Getting VxLAN interfaces'.format(host))
 705             inf_list = self.get_vxlan_interfaces(ip)
 706             for inf in inf_list:
 707                 deleted = True
 708                 time.sleep(2)
 709                 self.logger.info("{0}: Deleting VxLAN crossconnect {1}".format(host, inf))
 710                 url = self.vpp_inf_url.format(ip) + '/interface/' + inf + '/v3po:l2'
 711                 requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 712
 713             for inf in inf_list:
 714                 deleted = True
 715                 time.sleep(2)
 716                 self.logger.info("{0}: Deleting VxLAN interface {1}".format(host, inf))
 717                 url = self.vpp_inf_url.format(ip) + '/interface/' + inf
 718                 requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 719
 720             if self.get_vxlan_interfaces(ip):
 721                 self.logger.error("Error deleting VxLAN from {0}, try to restart the VM, IP is {1}.".format(host, ip))
 722                 return False
 723
 724             if not deleted:
 725                 self.logger.info("{0}: no VxLAN interface found, nothing to delete".format(host))
 726         return True
 727
 728     @staticmethod
 729     def save_object(obj, filepathname):
 730         with open(filepathname, 'wb') as fout:
 731             pickle.dump(obj, fout)
 732
 733     @staticmethod
 734     def load_object(filepathname):
 735         with open(filepathname, 'rb') as fin:
 736             return pickle.load(fin)
 737
 738     @staticmethod
 739     def increase_ip_address_or_vni_in_template(vnf_template_file, vnf_parameter_name_list):
 740         with open(vnf_template_file) as json_input:
 741             json_data = json.load(json_input)
 742             param_list = json_data['VNF-API:input']['VNF-API:vnf-topology-information']['VNF-API:vnf-parameters']
 743             for param in param_list:
 744                 if param['vnf-parameter-name'] in vnf_parameter_name_list:
 745                     ipaddr_or_vni = param['vnf-parameter-value'].split('.')
 746                     number = int(ipaddr_or_vni[-1])
 747                     if 254 == number:
 748                         number = 10
 749                     else:
 750                         number = number + 1
 751                     ipaddr_or_vni[-1] = str(number)
 752                     param['vnf-parameter-value'] = '.'.join(ipaddr_or_vni)
 753
 754         assert json_data is not None
 755         with open(vnf_template_file, 'w') as json_output:
 756             json.dump(json_data, json_output, indent=4, sort_keys=True)
 757
 758     def save_preload_data(self, preload_data):
 759         self.save_object(preload_data, self.preload_dict_file)
 760
 761     def load_preload_data(self):
 762         return self.load_object(self.preload_dict_file)
 763
 764     def save_vgmux_vnf_name(self, vgmux_vnf_name):
 765         self.save_object(vgmux_vnf_name, self.vgmux_vnf_name_file)
 766
 767     def load_vgmux_vnf_name(self):
 768         return self.load_object(self.vgmux_vnf_name_file)