test/vcpe/vcpecommon.py

   1 import json
   2 import logging
   3 import os
   4 import pickle
   5 import re
   6 import sys
   7
   8 import ipaddress
   9 import mysql.connector
  10 import requests
  11 import commands
  12 import time
  13 from novaclient import client as openstackclient
  14 from kubernetes import client, config
  15 from netaddr import IPAddress, IPNetwork
  16
  17 ######################################################################
  18 # Parts which must be updated / cross-checked during each deployment #
  19 # are marked as CHANGEME                                             #
  20 ######################################################################
  21
  22 class VcpeCommon:
  23     #############################################################################################
  24     # Set network prefix of k8s host external address; it's used for pod public IP autodetection
  25     # but can be overriden from user in case of autodetection failure
  26     external_net_addr = '10.12.0.0'
  27     external_net_prefix_len = 16
  28
  29     #############################################################################################
  30     # set the openstack cloud access credentials here
  31     oom_mode = True
  32
  33     ###########################
  34     # set Openstack credentials
  35     # CHANGEME part
  36     cloud = {
  37         '--os-auth-url': 'http://10.12.25.2:5000',
  38         '--os-username': 'kxi',
  39         '--os-user-domain-id': 'default',
  40         '--os-project-domain-id': 'default',
  41         '--os-tenant-id': 'bc43d50ffcb84750bac0c1707a9a765b' if oom_mode else '1e097c6713e74fd7ac8e4295e605ee1e',
  42         '--os-region-name': 'RegionOne',
  43         '--os-password': 'n3JhGMGuDzD8',
  44         '--os-project-domain-name': 'Integration-SB-03' if oom_mode else 'Integration-SB-07',
  45         '--os-identity-api-version': '3'
  46     }
  47
  48     ############################################################################
  49     # set oam and public network which must exist in openstack before deployment
  50     # CHANGEME part
  51     common_preload_config = {
  52         'oam_onap_net': 'oam_network_2No2' if oom_mode else 'oam_onap_lAky',
  53         'oam_onap_subnet': 'oam_network_2No2' if oom_mode else 'oam_onap_lAky',
  54         'public_net': 'external',
  55         'public_net_id': '971040b2-7059-49dc-b220-4fab50cb2ad4'
  56     }
  57
  58     #############################################################################
  59     # Set name of Onap's k8s namespace and sdnc controller pod
  60     # CHANGEME part
  61     onap_namespace = 'dev'
  62     sdnc_controller_pod = '-'.join([onap_namespace,'sdnc-sdnc-0'])
  63
  64     template_variable_symbol = '${'
  65     cpe_vm_prefix = 'zdcpe'
  66
  67     #############################################################################################
  68     # preloading network config
  69     #  key=network role
  70     #  value = [subnet_start_ip, subnet_gateway_ip]
  71     preload_network_config = {
  72         'cpe_public': ['10.2.0.2', '10.2.0.1'],
  73         'cpe_signal': ['10.4.0.2', '10.4.0.1'],
  74         'brg_bng': ['10.3.0.2', '10.3.0.1'],
  75         'bng_mux': ['10.1.0.10', '10.1.0.1'],
  76         'mux_gw': ['10.5.0.10', '10.5.0.1']
  77     }
  78
  79     dcae_ves_collector_name = 'dcae-bootstrap'
  80     global_subscriber_id = 'SDN-ETHERNET-INTERNET'
  81     project_name = 'Project-Demonstration'
  82     owning_entity_id = '520cc603-a3c4-4ec2-9ef4-ca70facd79c0'
  83     owning_entity_name = 'OE-Demonstration1'
  84
  85     def __init__(self, extra_host_names=None):
  86         self.logger = logging.getLogger(__name__)
  87         self.logger.setLevel(logging.DEBUG)
  88         self.logger.info('Initializing configuration')
  89
  90         ##################################################################################################################################
  91         # following param must be updated e.g. from csar file (grep for VfModuleModelInvariantUuid string) before vcpe.py customer call !!
  92         # vgw_VfModuleModelInvariantUuid is in rescust service csar,
  93         # look in service-VcpesvcRescust1118-template.yml for groups vgw module metadata. TODO: read this value automatically
  94         # CHANGEME part
  95         self.vgw_VfModuleModelInvariantUuid = '26d6a718-17b2-4ba8-8691-c44343b2ecd2'
  96
  97         # OOM: this is the address that the brg and bng will nat for sdnc access - 10.0.0.x address of k8 host for sdnc-0 container
  98         self.sdnc_oam_ip = self.get_pod_node_oam_ip(self.sdnc_controller_pod)
  99         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
 100         self.oom_so_sdnc_aai_ip = self.get_pod_node_public_ip(self.sdnc_controller_pod)
 101         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
 102         self.oom_dcae_ves_collector = self.oom_so_sdnc_aai_ip
 103         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
 104         self.mr_ip_addr = self.oom_so_sdnc_aai_ip
 105         self.mr_ip_port = '30227'
 106         self.so_nbi_port = '30277' if self.oom_mode else '8080'
 107         self.sdnc_preloading_port = '30267' if self.oom_mode else '8282'
 108         self.aai_query_port = '30233' if self.oom_mode else '8443'
 109         self.sniro_port = '30288' if self.oom_mode else '8080'
 110
 111         self.host_names = ['sdc', 'so', 'sdnc', 'robot', 'aai-inst1', self.dcae_ves_collector_name]
 112         if extra_host_names:
 113             self.host_names.extend(extra_host_names)
 114         # get IP addresses
 115         self.hosts = self.get_vm_ip(self.host_names, self.external_net_addr, self.external_net_prefix_len)
 116         # this is the keyword used to name vgw stack, must not be used in other stacks
 117         self.vgw_name_keyword = 'base_vcpe_vgw'
 118         # this is the file that will keep the index of last assigned SO name
 119         self.vgw_vfmod_name_index_file= '__var/vgw_vfmod_name_index'
 120         self.svc_instance_uuid_file = '__var/svc_instance_uuid'
 121         self.preload_dict_file = '__var/preload_dict'
 122         self.vgmux_vnf_name_file = '__var/vgmux_vnf_name'
 123         self.product_family_id = 'f9457e8c-4afd-45da-9389-46acd9bf5116'
 124         self.custom_product_family_id = 'a9a77d5a-123e-4ca2-9eb9-0b015d2ee0fb'
 125         self.instance_name_prefix = {
 126             'service': 'vcpe_svc',
 127             'network': 'vcpe_net',
 128             'vnf': 'vcpe_vnf',
 129             'vfmodule': 'vcpe_vfmodule'
 130         }
 131         self.aai_userpass = 'AAI', 'AAI'
 132
 133         ############################################################################################################
 134         # following key is overriding public key from vCPE heat templates, it's important to use correct one in here
 135         # CHANGEME part
 136         self.pub_key = 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKXDgoo3+WOqcUG8/5uUbk81+yczgwC4Y8ywTmuQqbNxlY1oQ0YxdMUqUnhitSXs5S/yRuAVOYHwGg2mCs20oAINrP+mxBI544AMIb9itPjCtgqtE2EWo6MmnFGbHB4Sx3XioE7F4VPsh7japsIwzOjbrQe+Mua1TGQ5d4nfEOQaaglXLLPFfuc7WbhbJbK6Q7rHqZfRcOwAMXgDoBqlyqKeiKwnumddo2RyNT8ljYmvB6buz7KnMinzo7qB0uktVT05FH9Rg0CTWH5norlG5qXgP2aukL0gk1ph8iAt7uYLf1ktp+LJI2gaF6L0/qli9EmVCSLr1uJ38Q8CBflhkh'
 137
 138         self.os_tenant_id = self.cloud['--os-tenant-id']
 139         self.os_region_name = self.cloud['--os-region-name']
 140         self.common_preload_config['pub_key'] = self.pub_key
 141         self.sniro_url = 'http://' + self.hosts['robot'] + ':' + self.sniro_port + '/__admin/mappings'
 142         self.sniro_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 143         self.homing_solution = 'sniro'  # value is either 'sniro' or 'oof'
 144 #        self.homing_solution = 'oof'
 145         self.customer_location_used_by_oof = {
 146             "customerLatitude": "32.897480",
 147             "customerLongitude": "-97.040443",
 148             "customerName": "some_company"
 149         }
 150
 151         #############################################################################################
 152         # SDC urls
 153         self.sdc_be_port = '30204'
 154         self.sdc_be_request_userpass = 'vid', 'Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U'
 155         self.sdc_be_request_headers = {'X-ECOMP-InstanceID': 'VID'}
 156         self.sdc_be_url_prefix = 'https://' + self.hosts['sdc'] + ':' + self.sdc_be_port
 157         self.sdc_service_list_url = self.sdc_be_url_prefix + '/sdc/v1/catalog/services'
 158
 159         self.sdc_fe_port = '30207'
 160         self.sdc_fe_request_userpass = 'beep', 'boop'
 161         self.sdc_fe_request_headers = {'USER_ID': 'demo', 'Content-Type': 'application/json'}
 162         self.sdc_fe_url_prefix = 'https://' + self.hosts['sdc'] + ':' + self.sdc_fe_port
 163         self.sdc_get_category_list_url = self.sdc_fe_url_prefix + '/sdc1/feProxy/rest/v1/categories'
 164         self.sdc_create_allotted_resource_subcategory_url = self.sdc_fe_url_prefix + '/sdc1/feProxy/rest/v1/category/resources/resourceNewCategory.allotted%20resource/subCategory'
 165
 166         #############################################################################################
 167         # SDNC urls
 168         self.sdnc_userpass = 'admin', 'Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U'
 169         self.sdnc_db_name = 'sdnctl'
 170         self.sdnc_db_user = 'sdnctl'
 171         self.sdnc_db_pass = 'gamma'
 172         self.sdnc_db_port = '32774'
 173         self.sdnc_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 174         self.sdnc_preload_network_url = 'https://' + self.hosts['sdnc'] + \
 175                                         ':' + self.sdnc_preloading_port + '/restconf/operations/VNF-API:preload-network-topology-operation'
 176         self.sdnc_preload_vnf_url = 'https://' + self.hosts['sdnc'] + \
 177                                     ':' + self.sdnc_preloading_port + '/restconf/operations/VNF-API:preload-vnf-topology-operation'
 178         self.sdnc_preload_gra_url = 'https://' + self.hosts['sdnc'] + \
 179                                     ':' + self.sdnc_preloading_port + '/restconf/operations/GENERIC-RESOURCE-API:preload-vf-module-topology-operation'
 180         self.sdnc_ar_cleanup_url = 'https://' + self.hosts['sdnc'] + ':' + self.sdnc_preloading_port + \
 181                                    '/restconf/config/GENERIC-RESOURCE-API:'
 182
 183         #############################################################################################
 184         # SO urls, note: do NOT add a '/' at the end of the url
 185         self.so_req_api_url = {'v4': 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/serviceInstantiation/v7/serviceInstances',
 186                            'v5': 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/serviceInstantiation/v7/serviceInstances'}
 187         self.so_check_progress_api_url = 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/orchestrationRequests/v6'
 188         self.so_userpass = 'InfraPortalClient', 'password1$'
 189         self.so_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 190         self.so_db_name = 'catalogdb'
 191         self.so_db_user = 'root'
 192         self.so_db_pass = 'secretpassword'
 193         self.so_db_port = '30252' if self.oom_mode else '32769'
 194
 195         self.vpp_inf_url = 'http://{0}:8183/restconf/config/ietf-interfaces:interfaces'
 196         self.vpp_api_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 197         self.vpp_api_userpass = ('admin', 'admin')
 198         self.vpp_ves_url= 'http://{0}:8183/restconf/config/vesagent:vesagent'
 199
 200         #############################################################################################
 201         # POLICY urls
 202         self.policy_userpass = ('healthcheck', 'zb!XztG34')
 203         self.policy_headers = {'Accept': 'application/json', 'Content-Type': 'application/json'}
 204         self.policy_api_url = 'https://{0}:6969/policy/api/v1/policytypes/onap.policies.controlloop.Operational/versions/1.0.0/policies'
 205         self.policy_pap_get_url = 'https://{0}:6969/policy/pap/v1/pdps'
 206         self.policy_pap_json = {'policies': [{'policy-id': 'operational.vcpe'}]}
 207         self.policy_pap_post_url = self.policy_pap_get_url + '/policies'
 208         self.policy_api_service_name = 'policy-api'
 209         self.policy_pap_service_name = 'policy-pap'
 210
 211     def heatbridge(self, openstack_stack_name, svc_instance_uuid):
 212         """
 213         Add vserver information to AAI
 214         """
 215         self.logger.info('Adding vServer information to AAI for {0}'.format(openstack_stack_name))
 216         if not self.oom_mode:
 217             cmd = '/opt/demo.sh heatbridge {0} {1} vCPE'.format(openstack_stack_name, svc_instance_uuid)
 218             ret = commands.getstatusoutput("ssh -i onap_dev root@{0} '{1}'".format(self.hosts['robot'], cmd))
 219             self.logger.debug('%s', ret)
 220         else:
 221             print('To add vGMUX vserver info to AAI, do the following:')
 222             print('- ssh to rancher')
 223             print('- sudo su -')
 224             print('- cd /root/oom/kubernetes/robot')
 225             print('- ./demo-k8s.sh onap heatbridge {0} {1} vCPE'.format(openstack_stack_name, svc_instance_uuid))
 226
 227     def get_brg_mac_from_sdnc(self):
 228         """
 229         Check table DHCP_MAP in the SDNC DB. Find the newly instantiated BRG MAC address.
 230         Note that there might be multiple BRGs, the most recently instantiated BRG always has the largest IP address.
 231         """
 232         cnx = mysql.connector.connect(user=self.sdnc_db_user, password=self.sdnc_db_pass, database=self.sdnc_db_name,
 233                                       host=self.hosts['sdnc'], port=self.sdnc_db_port)
 234         cursor = cnx.cursor()
 235         query = "SELECT * from DHCP_MAP"
 236         cursor.execute(query)
 237
 238         self.logger.debug('DHCP_MAP table in SDNC')
 239         mac_recent = None
 240         host = -1
 241         for mac, ip in cursor:
 242             self.logger.debug(mac + ':' + ip)
 243             this_host = int(ip.split('.')[-1])
 244             if host < this_host:
 245                 host = this_host
 246                 mac_recent = mac
 247
 248         cnx.close()
 249
 250         assert mac_recent
 251         return mac_recent
 252
 253     def execute_cmds_sdnc_db(self, cmds):
 254         self.execute_cmds_db(cmds, self.sdnc_db_user, self.sdnc_db_pass, self.sdnc_db_name,
 255                              self.hosts['sdnc'], self.sdnc_db_port)
 256
 257     def execute_cmds_so_db(self, cmds):
 258         self.execute_cmds_db(cmds, self.so_db_user, self.so_db_pass, self.so_db_name,
 259                              self.hosts['so'], self.so_db_port)
 260
 261     def execute_cmds_db(self, cmds, dbuser, dbpass, dbname, host, port):
 262         cnx = mysql.connector.connect(user=dbuser, password=dbpass, database=dbname, host=host, port=port)
 263         cursor = cnx.cursor()
 264         for cmd in cmds:
 265             self.logger.debug(cmd)
 266             cursor.execute(cmd)
 267             self.logger.debug('%s', cursor)
 268         cnx.commit()
 269         cursor.close()
 270         cnx.close()
 271
 272     def find_file(self, file_name_keyword, file_ext, search_dir):
 273         """
 274         :param file_name_keyword:  keyword used to look for the csar file, case insensitive matching, e.g, infra
 275         :param file_ext: e.g., csar, json
 276         :param search_dir path to search
 277         :return: path name of the file
 278         """
 279         file_name_keyword = file_name_keyword.lower()
 280         file_ext = file_ext.lower()
 281         if not file_ext.startswith('.'):
 282             file_ext = '.' + file_ext
 283
 284         filenamepath = None
 285         for file_name in os.listdir(search_dir):
 286             file_name_lower = file_name.lower()
 287             if file_name_keyword in file_name_lower and file_name_lower.endswith(file_ext):
 288                 if filenamepath:
 289                     self.logger.error('Multiple files found for *{0}*.{1} in '
 290                                       'directory {2}'.format(file_name_keyword, file_ext, search_dir))
 291                     sys.exit()
 292                 filenamepath = os.path.abspath(os.path.join(search_dir, file_name))
 293
 294         if filenamepath:
 295             return filenamepath
 296         else:
 297             self.logger.error("Cannot find *{0}*{1} in directory {2}".format(file_name_keyword, file_ext, search_dir))
 298             sys.exit()
 299
 300     @staticmethod
 301     def network_name_to_subnet_name(network_name):
 302         """
 303         :param network_name: example: vcpe_net_cpe_signal_201711281221
 304         :return: vcpe_net_cpe_signal_subnet_201711281221
 305         """
 306         fields = network_name.split('_')
 307         fields.insert(-1, 'subnet')
 308         return '_'.join(fields)
 309
 310     def set_network_name(self, network_name):
 311         param = ' '.join([k + ' ' + v for k, v in self.cloud.items()])
 312         openstackcmd = 'openstack ' + param
 313         cmd = ' '.join([openstackcmd, 'network set --name', network_name, 'ONAP-NW1'])
 314         os.popen(cmd)
 315
 316     def set_subnet_name(self, network_name):
 317         """
 318         Example: network_name =  vcpe_net_cpe_signal_201711281221
 319         set subnet name to vcpe_net_cpe_signal_subnet_201711281221
 320         :return:
 321         """
 322         param = ' '.join([k + ' ' + v for k, v in self.cloud.items()])
 323         openstackcmd = 'openstack ' + param
 324
 325         # expected results: | subnets | subnet_id |
 326         subnet_info = os.popen(openstackcmd + ' network show ' + network_name + ' |grep subnets').read().split('|')
 327         if len(subnet_info) > 2 and subnet_info[1].strip() == 'subnets':
 328             subnet_id = subnet_info[2].strip()
 329             subnet_name = self.network_name_to_subnet_name(network_name)
 330             cmd = ' '.join([openstackcmd, 'subnet set --name', subnet_name, subnet_id])
 331             os.popen(cmd)
 332             self.logger.info("Subnet name set to: " + subnet_name)
 333             return True
 334         else:
 335             self.logger.error("Can't get subnet info from network name: " + network_name)
 336             return False
 337
 338     def set_closed_loop_policy(self, policy_template_file):
 339         # Gather policy services cluster ips
 340         p_api_cluster_ip = self.get_k8s_service_cluster_ip(self.policy_api_service_name)
 341         p_pap_cluster_ip = self.get_k8s_service_cluster_ip(self.policy_pap_service_name)
 342
 343         # Read policy json from file
 344         with open(policy_template_file) as f:
 345             try:
 346                 policy_json = json.load(f)
 347             except ValueError:
 348                 self.logger.error(policy_template_file + " doesn't seem to contain valid JSON data")
 349                 sys.exit()
 350
 351         # Check policy already applied
 352         requests.packages.urllib3.disable_warnings()
 353         policy_exists_req = requests.get(self.policy_pap_get_url.format(
 354                             p_pap_cluster_ip), auth=self.policy_userpass,
 355                             verify=False, headers=self.policy_headers)
 356         if policy_exists_req.status_code != 200:
 357             self.logger.error('Failure in checking CL policy existence. '
 358                                'Policy-pap responded with HTTP code {0}'.format(
 359                                policy_exists_req.status_code))
 360             sys.exit()
 361
 362         try:
 363             policy_exists_json = policy_exists_req.json()
 364         except ValueError as e:
 365             self.logger.error('Policy-pap request failed: ' + e.message)
 366             sys.exit()
 367
 368         try:
 369             assert policy_exists_json['groups'][0]['pdpSubgroups'] \
 370                                [1]['policies'][0]['name'] != 'operational.vcpe'
 371         except AssertionError:
 372             self.logger.info('vCPE closed loop policy already exists, not applying')
 373             return
 374         except IndexError:
 375             pass # policy doesn't exist
 376
 377         # Create policy
 378         policy_create_req = requests.post(self.policy_api_url.format(
 379                             p_api_cluster_ip), auth=self.policy_userpass,
 380                             json=policy_json, verify=False,
 381                             headers=self.policy_headers)
 382         # Get the policy id from policy-api response
 383         if policy_create_req.status_code != 200:
 384             self.logger.error('Failed creating policy. Policy-api responded'
 385                               ' with HTTP code {0}'.format(policy_create_req.status_code))
 386             sys.exit()
 387
 388         try:
 389             policy_version = json.loads(policy_create_req.text)['policy-version']
 390         except (KeyError, ValueError):
 391             self.logger.error('Policy API response not understood:')
 392             self.logger.debug('\n' + str(policy_create_req.text))
 393
 394         # Inject the policy into Policy PAP
 395         self.policy_pap_json['policies'].append({'policy-version': policy_version})
 396         policy_insert_req = requests.post(self.policy_pap_post_url.format(
 397                             p_pap_cluster_ip), auth=self.policy_userpass,
 398                             json=self.policy_pap_json, verify=False,
 399                             headers=self.policy_headers)
 400         if policy_insert_req.status_code != 200:
 401             self.logger.error('Policy PAP request failed with HTTP code'
 402                               '{0}'.format(policy_insert_req.status_code))
 403             sys.exit()
 404         self.logger.info('Successully pushed closed loop Policy')
 405
 406     def is_node_in_aai(self, node_type, node_uuid):
 407         key = None
 408         search_node_type = None
 409         if node_type == 'service':
 410             search_node_type = 'service-instance'
 411             key = 'service-instance-id'
 412         elif node_type == 'vnf':
 413             search_node_type = 'generic-vnf'
 414             key = 'vnf-id'
 415         else:
 416             logging.error('Invalid node_type: ' + node_type)
 417             sys.exit()
 418
 419         url = 'https://{0}:{1}/aai/v11/search/nodes-query?search-node-type={2}&filter={3}:EQUALS:{4}'.format(
 420             self.hosts['aai-inst1'], self.aai_query_port, search_node_type, key, node_uuid)
 421
 422         headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'X-FromAppID': 'vCPE-Robot', 'X-TransactionId': 'get_aai_subscr'}
 423         requests.packages.urllib3.disable_warnings()
 424         r = requests.get(url, headers=headers, auth=self.aai_userpass, verify=False)
 425         response = r.json()
 426         self.logger.debug('aai query: ' + url)
 427         self.logger.debug('aai response:\n' + json.dumps(response, indent=4, sort_keys=True))
 428         return 'result-data' in response
 429
 430     @staticmethod
 431     def extract_ip_from_str(net_addr, net_addr_len, sz):
 432         """
 433         :param net_addr:  e.g. 10.5.12.0
 434         :param net_addr_len: e.g. 24
 435         :param sz: a string
 436         :return: the first IP address matching the network, e.g. 10.5.12.3
 437         """
 438         network = ipaddress.ip_network(unicode('{0}/{1}'.format(net_addr, net_addr_len)), strict=False)
 439         ip_list = re.findall(r'[0-9]+(?:\.[0-9]+){3}', sz)
 440         for ip in ip_list:
 441             this_net = ipaddress.ip_network(unicode('{0}/{1}'.format(ip, net_addr_len)), strict=False)
 442             if this_net == network:
 443                 return str(ip)
 444         return None
 445
 446     def get_pod_node_oam_ip(self, pod):
 447         """
 448         :Assuming kubectl is available and configured by default config (~/.kube/config)
 449         :param pod: pod name substring, e.g. 'sdnc-sdnc-0'
 450         :return pod's cluster node oam ip (10.0.0.0/16)
 451         """
 452         ret = None
 453         config.load_kube_config()
 454         api = client.CoreV1Api()
 455         kslogger = logging.getLogger('kubernetes')
 456         kslogger.setLevel(logging.INFO)
 457         res = api.list_pod_for_all_namespaces()
 458         for i in res.items:
 459             if pod in i.metadata.name:
 460                 self.logger.debug("found {0}\t{1}\t{2}".format(i.metadata.name, i.status.host_ip, i.spec.node_name))
 461                 ret = i.status.host_ip
 462                 break
 463
 464         if ret is None:
 465             ret = raw_input("Enter " + self.sdnc_controller_pod + " pod cluster node OAM IP address(10.0.0.0/16): ")
 466         return ret
 467
 468     def get_pod_node_public_ip(self, pod):
 469         """
 470         :Assuming kubectl is available and configured by default config (~/.kube/config)
 471         :param pod: pod name substring, e.g. 'sdnc-sdnc-0'
 472         :return pod's cluster node public ip (i.e. 10.12.0.0/16)
 473         """
 474         ret = None
 475         config.load_kube_config()
 476         api = client.CoreV1Api()
 477         kslogger = logging.getLogger('kubernetes')
 478         kslogger.setLevel(logging.INFO)
 479         res = api.list_pod_for_all_namespaces()
 480         for i in res.items:
 481             if pod in i.metadata.name:
 482                 ret = self.get_vm_public_ip_by_nova(i.spec.node_name)
 483                 self.logger.debug("found node {0} public ip: {1}".format(i.spec.node_name, ret))
 484                 break
 485
 486         if ret is None:
 487             ret = raw_input("Enter " + self.sdnc_controller_pod + " pod cluster node public IP address(i.e. " + self.external_net_addr + "): ")
 488         return ret
 489
 490     def get_vm_public_ip_by_nova(self, vm):
 491         """
 492         This method uses openstack nova api to retrieve vm public ip
 493         :param vm: vm name
 494         :return vm public ip
 495         """
 496         subnet = IPNetwork('{0}/{1}'.format(self.external_net_addr, self.external_net_prefix_len))
 497         nova = openstackclient.Client(2, self.cloud['--os-username'], self.cloud['--os-password'], self.cloud['--os-tenant-id'], self.cloud['--os-auth-url'])
 498         for i in nova.servers.list():
 499             if i.name == vm:
 500                 for k, v in i.networks.items():
 501                     for ip in v:
 502                         if IPAddress(ip) in subnet:
 503                             return ip
 504         return None
 505
 506     def get_vm_ip(self, keywords, net_addr=None, net_addr_len=None):
 507         """
 508         :param keywords: list of keywords to search for vm, e.g. ['bng', 'gmux', 'brg']
 509         :param net_addr: e.g. 10.12.5.0
 510         :param net_addr_len: e.g. 24
 511         :return: dictionary {keyword: ip}
 512         """
 513         if not net_addr:
 514             net_addr = self.external_net_addr
 515
 516         if not net_addr_len:
 517             net_addr_len = self.external_net_prefix_len
 518
 519         param = ' '.join([k + ' ' + v for k, v in self.cloud.items() if 'identity' not in k])
 520         openstackcmd = 'nova ' + param + ' list'
 521         self.logger.debug(openstackcmd)
 522
 523         results = os.popen(openstackcmd).read()
 524         all_vm_ip_dict = self.extract_vm_ip_as_dict(results, net_addr, net_addr_len)
 525         latest_vm_list = self.remove_old_vms(all_vm_ip_dict.keys(), self.cpe_vm_prefix)
 526         latest_vm_ip_dict = {vm: all_vm_ip_dict[vm] for vm in latest_vm_list}
 527         ip_dict = self.select_subset_vm_ip(latest_vm_ip_dict, keywords)
 528         if self.oom_mode:
 529             ip_dict.update(self.get_oom_onap_vm_ip(keywords))
 530
 531         if len(ip_dict) != len(keywords):
 532             self.logger.error('Cannot find all desired IP addresses for %s.', keywords)
 533             self.logger.error(json.dumps(ip_dict, indent=4, sort_keys=True))
 534             self.logger.error('Temporarily continue.. remember to check back vcpecommon.py line: 396')
 535 #            sys.exit()
 536         return ip_dict
 537
 538     def get_oom_onap_vm_ip(self, keywords):
 539         vm_ip = {}
 540         onap_vm_list = set(['sdc', 'so', 'sdnc', 'aai-inst1', 'robot', self.dcae_ves_collector_name])
 541         for vm in keywords:
 542             if vm in onap_vm_list:
 543                 vm_ip[vm] = self.oom_so_sdnc_aai_ip
 544         return vm_ip
 545
 546     def get_k8s_service_cluster_ip(self, service):
 547         """
 548         Returns cluster IP for a given service
 549         :param service: name of the service
 550         :return: cluster ip
 551         """
 552         config.load_kube_config()
 553         api = client.CoreV1Api()
 554         kslogger = logging.getLogger('kubernetes')
 555         kslogger.setLevel(logging.INFO)
 556         try:
 557             resp = api.read_namespaced_service(service, self.onap_namespace)
 558         except client.rest.ApiException as e:
 559             self.logger.error('Error while making k8s API request: ' + e.body)
 560             sys.exit()
 561
 562         return resp.spec.cluster_ip
 563
 564     def extract_vm_ip_as_dict(self, novalist_results, net_addr, net_addr_len):
 565         vm_ip_dict = {}
 566         for line in novalist_results.split('\n'):
 567             fields = line.split('|')
 568             if len(fields) == 8:
 569                 vm_name = fields[2]
 570                 ip_info = fields[-2]
 571                 ip = self.extract_ip_from_str(net_addr, net_addr_len, ip_info)
 572                 vm_ip_dict[vm_name] = ip
 573
 574         return vm_ip_dict
 575
 576     def remove_old_vms(self, vm_list, prefix):
 577         """
 578         For vms with format name_timestamp, only keep the one with the latest timestamp.
 579         E.g.,
 580             zdcpe1cpe01brgemu01_201805222148        (drop this)
 581             zdcpe1cpe01brgemu01_201805222229        (keep this)
 582             zdcpe1cpe01gw01_201805162201
 583         """
 584         new_vm_list = []
 585         same_type_vm_dict = {}
 586         for vm in vm_list:
 587             fields = vm.split('_')
 588             if vm.startswith(prefix) and len(fields) == 2 and len(fields[-1]) == len('201805222148') and fields[-1].isdigit():
 589                 if vm > same_type_vm_dict.get(fields[0], '0'):
 590                     same_type_vm_dict[fields[0]] = vm
 591             else:
 592                 new_vm_list.append(vm)
 593
 594         new_vm_list.extend(same_type_vm_dict.values())
 595         return new_vm_list
 596
 597     def select_subset_vm_ip(self, all_vm_ip_dict, vm_name_keyword_list):
 598         vm_ip_dict = {}
 599         for keyword in vm_name_keyword_list:
 600             for vm, ip in all_vm_ip_dict.items():
 601                 if keyword in vm:
 602                     vm_ip_dict[keyword] = ip
 603                     break
 604         return vm_ip_dict
 605
 606     def del_vgmux_ves_mode(self):
 607         url = self.vpp_ves_url.format(self.hosts['mux']) + '/mode'
 608         r = requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 609         self.logger.debug('%s', r)
 610
 611     def del_vgmux_ves_collector(self):
 612         url = self.vpp_ves_url.format(self.hosts['mux']) + '/config'
 613         r = requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 614         self.logger.debug('%s', r)
 615
 616     def set_vgmux_ves_collector(self ):
 617         url = self.vpp_ves_url.format(self.hosts['mux'])
 618         data = {'config':
 619                     {'server-addr': self.hosts[self.dcae_ves_collector_name],
 620                      'server-port': '30235' if self.oom_mode else '8081',
 621                      'read-interval': '10',
 622                      'is-add':'1'
 623                      }
 624                 }
 625         r = requests.post(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass, json=data)
 626         self.logger.debug('%s', r)
 627
 628     def set_vgmux_packet_loss_rate(self, lossrate, vg_vnf_instance_name):
 629         url = self.vpp_ves_url.format(self.hosts['mux'])
 630         data = {"mode":
 631                     {"working-mode": "demo",
 632                      "base-packet-loss": str(lossrate),
 633                      "source-name": vg_vnf_instance_name
 634                      }
 635                 }
 636         r = requests.post(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass, json=data)
 637         self.logger.debug('%s', r)
 638
 639         # return all the VxLAN interface names of BRG or vGMUX based on the IP address
 640     def get_vxlan_interfaces(self, ip, print_info=False):
 641         url = self.vpp_inf_url.format(ip)
 642         self.logger.debug('url is this: %s', url)
 643         r = requests.get(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 644         data = r.json()['interfaces']['interface']
 645         if print_info:
 646             for inf in data:
 647                 if 'name' in inf and 'type' in inf and inf['type'] == 'v3po:vxlan-tunnel':
 648                     print(json.dumps(inf, indent=4, sort_keys=True))
 649
 650         return [inf['name'] for inf in data if 'name' in inf and 'type' in inf and inf['type'] == 'v3po:vxlan-tunnel']
 651
 652     # delete all VxLAN interfaces of each hosts
 653     def delete_vxlan_interfaces(self, host_dic):
 654         for host, ip in host_dic.items():
 655             deleted = False
 656             self.logger.info('{0}: Getting VxLAN interfaces'.format(host))
 657             inf_list = self.get_vxlan_interfaces(ip)
 658             for inf in inf_list:
 659                 deleted = True
 660                 time.sleep(2)
 661                 self.logger.info("{0}: Deleting VxLAN crossconnect {1}".format(host, inf))
 662                 url = self.vpp_inf_url.format(ip) + '/interface/' + inf + '/v3po:l2'
 663                 requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 664
 665             for inf in inf_list:
 666                 deleted = True
 667                 time.sleep(2)
 668                 self.logger.info("{0}: Deleting VxLAN interface {1}".format(host, inf))
 669                 url = self.vpp_inf_url.format(ip) + '/interface/' + inf
 670                 requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 671
 672             if len(self.get_vxlan_interfaces(ip)) > 0:
 673                 self.logger.error("Error deleting VxLAN from {0}, try to restart the VM, IP is {1}.".format(host, ip))
 674                 return False
 675
 676             if not deleted:
 677                 self.logger.info("{0}: no VxLAN interface found, nothing to delete".format(host))
 678         return True
 679
 680     @staticmethod
 681     def save_object(obj, filepathname):
 682         with open(filepathname, 'wb') as fout:
 683             pickle.dump(obj, fout)
 684
 685     @staticmethod
 686     def load_object(filepathname):
 687         with open(filepathname, 'rb') as fin:
 688             return pickle.load(fin)
 689
 690     @staticmethod
 691     def increase_ip_address_or_vni_in_template(vnf_template_file, vnf_parameter_name_list):
 692         with open(vnf_template_file) as json_input:
 693             json_data = json.load(json_input)
 694             param_list = json_data['VNF-API:input']['VNF-API:vnf-topology-information']['VNF-API:vnf-parameters']
 695             for param in param_list:
 696                 if param['vnf-parameter-name'] in vnf_parameter_name_list:
 697                     ipaddr_or_vni = param['vnf-parameter-value'].split('.')
 698                     number = int(ipaddr_or_vni[-1])
 699                     if 254 == number:
 700                         number = 10
 701                     else:
 702                         number = number + 1
 703                     ipaddr_or_vni[-1] = str(number)
 704                     param['vnf-parameter-value'] = '.'.join(ipaddr_or_vni)
 705
 706         assert json_data is not None
 707         with open(vnf_template_file, 'w') as json_output:
 708             json.dump(json_data, json_output, indent=4, sort_keys=True)
 709
 710     def save_preload_data(self, preload_data):
 711         self.save_object(preload_data, self.preload_dict_file)
 712
 713     def load_preload_data(self):
 714         return self.load_object(self.preload_dict_file)
 715
 716     def save_vgmux_vnf_name(self, vgmux_vnf_name):
 717         self.save_object(vgmux_vnf_name, self.vgmux_vnf_name_file)
 718
 719     def load_vgmux_vnf_name(self):
 720         return self.load_object(self.vgmux_vnf_name_file)
 721