test/vcpe/vcpecommon.py

   1 #!/usr/bin/env python
   2
   3 import json
   4 import logging
   5 import os
   6 import pickle
   7 import re
   8 import sys
   9
  10 import ipaddress
  11 import mysql.connector
  12 import requests
  13 import commands
  14 import time
  15 from novaclient import client as openstackclient
  16 from kubernetes import client, config
  17 from netaddr import IPAddress, IPNetwork
  18
  19 ######################################################################
  20 # Parts which must be updated / cross-checked during each deployment #
  21 # are marked as CHANGEME                                             #
  22 ######################################################################
  23
  24 class VcpeCommon:
  25     #############################################################################################
  26     # Set network prefix of k8s host external address; it's used for pod public IP autodetection
  27     # but can be overriden from user in case of autodetection failure
  28     external_net_addr = '10.12.0.0'
  29     external_net_prefix_len = 16
  30
  31     #############################################################################################
  32     # set the openstack cloud access credentials here
  33     oom_mode = True
  34
  35     ###########################
  36     # set Openstack credentials
  37     # CHANGEME part
  38     cloud = {
  39         '--os-auth-url': 'http://10.12.25.2:5000',
  40         '--os-username': 'kxi',
  41         '--os-user-domain-id': 'default',
  42         '--os-project-domain-id': 'default',
  43         '--os-tenant-id': 'bc43d50ffcb84750bac0c1707a9a765b' if oom_mode else '1e097c6713e74fd7ac8e4295e605ee1e',
  44         '--os-region-name': 'RegionOne',
  45         '--os-password': 'n3JhGMGuDzD8',
  46         '--os-project-domain-name': 'Integration-SB-03' if oom_mode else 'Integration-SB-07',
  47         '--os-identity-api-version': '3'
  48     }
  49
  50     ############################################################################
  51     # set oam and public network which must exist in openstack before deployment
  52     # CHANGEME part
  53     common_preload_config = {
  54         'oam_onap_net': 'oam_network_2No2' if oom_mode else 'oam_onap_lAky',
  55         'oam_onap_subnet': 'oam_network_2No2' if oom_mode else 'oam_onap_lAky',
  56         'public_net': 'external',
  57         'public_net_id': '971040b2-7059-49dc-b220-4fab50cb2ad4'
  58     }
  59
  60     #############################################################################
  61     # Set name of Onap's k8s namespace and sdnc controller pod
  62     # CHANGEME part
  63     onap_namespace = 'onap'
  64     onap_environment = 'dev'
  65     sdnc_controller_pod = '-'.join([onap_environment, 'sdnc-sdnc-0'])
  66
  67     template_variable_symbol = '${'
  68     cpe_vm_prefix = 'zdcpe'
  69
  70     #############################################################################################
  71     # preloading network config
  72     #  key=network role
  73     #  value = [subnet_start_ip, subnet_gateway_ip]
  74     preload_network_config = {
  75         'cpe_public': ['10.2.0.2', '10.2.0.1'],
  76         'cpe_signal': ['10.4.0.2', '10.4.0.1'],
  77         'brg_bng': ['10.3.0.2', '10.3.0.1'],
  78         'bng_mux': ['10.1.0.10', '10.1.0.1'],
  79         'mux_gw': ['10.5.0.10', '10.5.0.1']
  80     }
  81
  82     dcae_ves_collector_name = 'dcae-bootstrap'
  83     global_subscriber_id = 'SDN-ETHERNET-INTERNET'
  84     project_name = 'Project-Demonstration'
  85     owning_entity_id = '520cc603-a3c4-4ec2-9ef4-ca70facd79c0'
  86     owning_entity_name = 'OE-Demonstration1'
  87
  88     def __init__(self, extra_host_names=None):
  89         self.logger = logging.getLogger(__name__)
  90         self.logger.setLevel(logging.DEBUG)
  91         self.logger.info('Initializing configuration')
  92
  93         ##################################################################################################################################
  94         # following param must be updated e.g. from csar file (grep for VfModuleModelInvariantUuid string) before vcpe.py customer call !!
  95         # vgw_VfModuleModelInvariantUuid is in rescust service csar,
  96         # look in service-VcpesvcRescust1118-template.yml for groups vgw module metadata. TODO: read this value automatically
  97         # CHANGEME part
  98         self.vgw_VfModuleModelInvariantUuid = '26d6a718-17b2-4ba8-8691-c44343b2ecd2'
  99
 100         # OOM: this is the address that the brg and bng will nat for sdnc access - 10.0.0.x address of k8 host for sdnc-0 container
 101         self.sdnc_oam_ip = self.get_pod_node_oam_ip(self.sdnc_controller_pod)
 102         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
 103         self.oom_so_sdnc_aai_ip = self.get_pod_node_public_ip(self.sdnc_controller_pod)
 104         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
 105         self.oom_dcae_ves_collector = self.oom_so_sdnc_aai_ip
 106         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
 107         self.mr_ip_addr = self.oom_so_sdnc_aai_ip
 108         self.mr_ip_port = '30227'
 109         self.so_nbi_port = '30277' if self.oom_mode else '8080'
 110         self.sdnc_preloading_port = '30267' if self.oom_mode else '8282'
 111         self.aai_query_port = '30233' if self.oom_mode else '8443'
 112         self.sniro_port = '30288' if self.oom_mode else '8080'
 113
 114         self.host_names = ['sdc', 'so', 'sdnc', 'robot', 'aai-inst1', self.dcae_ves_collector_name]
 115         if extra_host_names:
 116             self.host_names.extend(extra_host_names)
 117         # get IP addresses
 118         self.hosts = self.get_vm_ip(self.host_names, self.external_net_addr, self.external_net_prefix_len)
 119         # this is the keyword used to name vgw stack, must not be used in other stacks
 120         self.vgw_name_keyword = 'base_vcpe_vgw'
 121         # this is the file that will keep the index of last assigned SO name
 122         self.vgw_vfmod_name_index_file= '__var/vgw_vfmod_name_index'
 123         self.svc_instance_uuid_file = '__var/svc_instance_uuid'
 124         self.preload_dict_file = '__var/preload_dict'
 125         self.vgmux_vnf_name_file = '__var/vgmux_vnf_name'
 126         self.product_family_id = 'f9457e8c-4afd-45da-9389-46acd9bf5116'
 127         self.custom_product_family_id = 'a9a77d5a-123e-4ca2-9eb9-0b015d2ee0fb'
 128         self.instance_name_prefix = {
 129             'service': 'vcpe_svc',
 130             'network': 'vcpe_net',
 131             'vnf': 'vcpe_vnf',
 132             'vfmodule': 'vcpe_vfmodule'
 133         }
 134         self.aai_userpass = 'AAI', 'AAI'
 135
 136         ############################################################################################################
 137         # following key is overriding public key from vCPE heat templates, it's important to use correct one in here
 138         # CHANGEME part
 139         self.pub_key = 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKXDgoo3+WOqcUG8/5uUbk81+yczgwC4Y8ywTmuQqbNxlY1oQ0YxdMUqUnhitSXs5S/yRuAVOYHwGg2mCs20oAINrP+mxBI544AMIb9itPjCtgqtE2EWo6MmnFGbHB4Sx3XioE7F4VPsh7japsIwzOjbrQe+Mua1TGQ5d4nfEOQaaglXLLPFfuc7WbhbJbK6Q7rHqZfRcOwAMXgDoBqlyqKeiKwnumddo2RyNT8ljYmvB6buz7KnMinzo7qB0uktVT05FH9Rg0CTWH5norlG5qXgP2aukL0gk1ph8iAt7uYLf1ktp+LJI2gaF6L0/qli9EmVCSLr1uJ38Q8CBflhkh'
 140
 141         self.os_tenant_id = self.cloud['--os-tenant-id']
 142         self.os_region_name = self.cloud['--os-region-name']
 143         self.common_preload_config['pub_key'] = self.pub_key
 144         self.sniro_url = 'http://' + self.hosts['robot'] + ':' + self.sniro_port + '/__admin/mappings'
 145         self.sniro_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 146         self.homing_solution = 'sniro'  # value is either 'sniro' or 'oof'
 147 #        self.homing_solution = 'oof'
 148         self.customer_location_used_by_oof = {
 149             "customerLatitude": "32.897480",
 150             "customerLongitude": "-97.040443",
 151             "customerName": "some_company"
 152         }
 153
 154         #############################################################################################
 155         # SDC urls
 156         self.sdc_be_port = '30204'
 157         self.sdc_be_request_userpass = 'vid', 'Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U'
 158         self.sdc_be_request_headers = {'X-ECOMP-InstanceID': 'VID'}
 159         self.sdc_be_url_prefix = 'https://' + self.hosts['sdc'] + ':' + self.sdc_be_port
 160         self.sdc_service_list_url = self.sdc_be_url_prefix + '/sdc/v1/catalog/services'
 161
 162         self.sdc_fe_port = '30207'
 163         self.sdc_fe_request_userpass = 'beep', 'boop'
 164         self.sdc_fe_request_headers = {'USER_ID': 'demo', 'Content-Type': 'application/json'}
 165         self.sdc_fe_url_prefix = 'https://' + self.hosts['sdc'] + ':' + self.sdc_fe_port
 166         self.sdc_get_category_list_url = self.sdc_fe_url_prefix + '/sdc1/feProxy/rest/v1/categories'
 167         self.sdc_create_allotted_resource_subcategory_url = self.sdc_fe_url_prefix + '/sdc1/feProxy/rest/v1/category/resources/resourceNewCategory.allotted%20resource/subCategory'
 168
 169         #############################################################################################
 170         # SDNC urls
 171         self.sdnc_userpass = 'admin', 'Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U'
 172         self.sdnc_db_name = 'sdnctl'
 173         self.sdnc_db_user = 'sdnctl'
 174         self.sdnc_db_pass = 'gamma'
 175         self.sdnc_db_port = '32774'
 176         self.sdnc_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 177         self.sdnc_preload_network_url = 'https://' + self.hosts['sdnc'] + \
 178                                         ':' + self.sdnc_preloading_port + '/restconf/operations/VNF-API:preload-network-topology-operation'
 179         self.sdnc_preload_vnf_url = 'https://' + self.hosts['sdnc'] + \
 180                                     ':' + self.sdnc_preloading_port + '/restconf/operations/VNF-API:preload-vnf-topology-operation'
 181         self.sdnc_preload_gra_url = 'https://' + self.hosts['sdnc'] + \
 182                                     ':' + self.sdnc_preloading_port + '/restconf/operations/GENERIC-RESOURCE-API:preload-vf-module-topology-operation'
 183         self.sdnc_ar_cleanup_url = 'https://' + self.hosts['sdnc'] + ':' + self.sdnc_preloading_port + \
 184                                    '/restconf/config/GENERIC-RESOURCE-API:'
 185
 186         #############################################################################################
 187         # SO urls, note: do NOT add a '/' at the end of the url
 188         self.so_req_api_url = {'v4': 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/serviceInstantiation/v7/serviceInstances',
 189                            'v5': 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/serviceInstantiation/v7/serviceInstances'}
 190         self.so_check_progress_api_url = 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/orchestrationRequests/v6'
 191         self.so_userpass = 'InfraPortalClient', 'password1$'
 192         self.so_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 193         self.so_db_name = 'catalogdb'
 194         self.so_db_user = 'root'
 195         self.so_db_pass = 'secretpassword'
 196         self.so_db_port = '30252' if self.oom_mode else '32769'
 197
 198         self.vpp_inf_url = 'http://{0}:8183/restconf/config/ietf-interfaces:interfaces'
 199         self.vpp_api_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 200         self.vpp_api_userpass = ('admin', 'admin')
 201         self.vpp_ves_url= 'http://{0}:8183/restconf/config/vesagent:vesagent'
 202
 203         #############################################################################################
 204         # POLICY urls
 205         self.policy_userpass = ('healthcheck', 'zb!XztG34')
 206         self.policy_headers = {'Accept': 'application/json', 'Content-Type': 'application/json'}
 207         self.policy_api_url = 'https://{0}:6969/policy/api/v1/policytypes/onap.policies.controlloop.Operational/versions/1.0.0/policies'
 208         self.policy_pap_get_url = 'https://{0}:6969/policy/pap/v1/pdps'
 209         self.policy_pap_json = {'policies': [{'policy-id': 'operational.vcpe'}]}
 210         self.policy_pap_post_url = self.policy_pap_get_url + '/policies'
 211         self.policy_api_service_name = 'policy-api'
 212         self.policy_pap_service_name = 'policy-pap'
 213
 214     def heatbridge(self, openstack_stack_name, svc_instance_uuid):
 215         """
 216         Add vserver information to AAI
 217         """
 218         self.logger.info('Adding vServer information to AAI for {0}'.format(openstack_stack_name))
 219         if not self.oom_mode:
 220             cmd = '/opt/demo.sh heatbridge {0} {1} vCPE'.format(openstack_stack_name, svc_instance_uuid)
 221             ret = commands.getstatusoutput("ssh -i onap_dev root@{0} '{1}'".format(self.hosts['robot'], cmd))
 222             self.logger.debug('%s', ret)
 223         else:
 224             print('To add vGMUX vserver info to AAI, do the following:')
 225             print('- ssh to rancher')
 226             print('- sudo su -')
 227             print('- cd /root/oom/kubernetes/robot')
 228             print('- ./demo-k8s.sh onap heatbridge {0} {1} vCPE'.format(openstack_stack_name, svc_instance_uuid))
 229
 230     def get_brg_mac_from_sdnc(self):
 231         """
 232         Check table DHCP_MAP in the SDNC DB. Find the newly instantiated BRG MAC address.
 233         Note that there might be multiple BRGs, the most recently instantiated BRG always has the largest IP address.
 234         """
 235         cnx = mysql.connector.connect(user=self.sdnc_db_user, password=self.sdnc_db_pass, database=self.sdnc_db_name,
 236                                       host=self.hosts['sdnc'], port=self.sdnc_db_port)
 237         cursor = cnx.cursor()
 238         query = "SELECT * from DHCP_MAP"
 239         cursor.execute(query)
 240
 241         self.logger.debug('DHCP_MAP table in SDNC')
 242         mac_recent = None
 243         host = -1
 244         for mac, ip in cursor:
 245             self.logger.debug(mac + ':' + ip)
 246             this_host = int(ip.split('.')[-1])
 247             if host < this_host:
 248                 host = this_host
 249                 mac_recent = mac
 250
 251         cnx.close()
 252
 253         assert mac_recent
 254         return mac_recent
 255
 256     def execute_cmds_sdnc_db(self, cmds):
 257         self.execute_cmds_db(cmds, self.sdnc_db_user, self.sdnc_db_pass, self.sdnc_db_name,
 258                              self.hosts['sdnc'], self.sdnc_db_port)
 259
 260     def execute_cmds_so_db(self, cmds):
 261         self.execute_cmds_db(cmds, self.so_db_user, self.so_db_pass, self.so_db_name,
 262                              self.hosts['so'], self.so_db_port)
 263
 264     def execute_cmds_db(self, cmds, dbuser, dbpass, dbname, host, port):
 265         cnx = mysql.connector.connect(user=dbuser, password=dbpass, database=dbname, host=host, port=port)
 266         cursor = cnx.cursor()
 267         for cmd in cmds:
 268             self.logger.debug(cmd)
 269             cursor.execute(cmd)
 270             self.logger.debug('%s', cursor)
 271         cnx.commit()
 272         cursor.close()
 273         cnx.close()
 274
 275     def find_file(self, file_name_keyword, file_ext, search_dir):
 276         """
 277         :param file_name_keyword:  keyword used to look for the csar file, case insensitive matching, e.g, infra
 278         :param file_ext: e.g., csar, json
 279         :param search_dir path to search
 280         :return: path name of the file
 281         """
 282         file_name_keyword = file_name_keyword.lower()
 283         file_ext = file_ext.lower()
 284         if not file_ext.startswith('.'):
 285             file_ext = '.' + file_ext
 286
 287         filenamepath = None
 288         for file_name in os.listdir(search_dir):
 289             file_name_lower = file_name.lower()
 290             if file_name_keyword in file_name_lower and file_name_lower.endswith(file_ext):
 291                 if filenamepath:
 292                     self.logger.error('Multiple files found for *{0}*.{1} in '
 293                                       'directory {2}'.format(file_name_keyword, file_ext, search_dir))
 294                     sys.exit()
 295                 filenamepath = os.path.abspath(os.path.join(search_dir, file_name))
 296
 297         if filenamepath:
 298             return filenamepath
 299         else:
 300             self.logger.error("Cannot find *{0}*{1} in directory {2}".format(file_name_keyword, file_ext, search_dir))
 301             sys.exit()
 302
 303     @staticmethod
 304     def network_name_to_subnet_name(network_name):
 305         """
 306         :param network_name: example: vcpe_net_cpe_signal_201711281221
 307         :return: vcpe_net_cpe_signal_subnet_201711281221
 308         """
 309         fields = network_name.split('_')
 310         fields.insert(-1, 'subnet')
 311         return '_'.join(fields)
 312
 313     def set_network_name(self, network_name):
 314         param = ' '.join([k + ' ' + v for k, v in self.cloud.items()])
 315         openstackcmd = 'openstack ' + param
 316         cmd = ' '.join([openstackcmd, 'network set --name', network_name, 'ONAP-NW1'])
 317         os.popen(cmd)
 318
 319     def set_subnet_name(self, network_name):
 320         """
 321         Example: network_name =  vcpe_net_cpe_signal_201711281221
 322         set subnet name to vcpe_net_cpe_signal_subnet_201711281221
 323         :return:
 324         """
 325         param = ' '.join([k + ' ' + v for k, v in self.cloud.items()])
 326         openstackcmd = 'openstack ' + param
 327
 328         # expected results: | subnets | subnet_id |
 329         subnet_info = os.popen(openstackcmd + ' network show ' + network_name + ' |grep subnets').read().split('|')
 330         if len(subnet_info) > 2 and subnet_info[1].strip() == 'subnets':
 331             subnet_id = subnet_info[2].strip()
 332             subnet_name = self.network_name_to_subnet_name(network_name)
 333             cmd = ' '.join([openstackcmd, 'subnet set --name', subnet_name, subnet_id])
 334             os.popen(cmd)
 335             self.logger.info("Subnet name set to: " + subnet_name)
 336             return True
 337         else:
 338             self.logger.error("Can't get subnet info from network name: " + network_name)
 339             return False
 340
 341     def set_closed_loop_policy(self, policy_template_file):
 342         # Gather policy services cluster ips
 343         p_api_cluster_ip = self.get_k8s_service_cluster_ip(self.policy_api_service_name)
 344         p_pap_cluster_ip = self.get_k8s_service_cluster_ip(self.policy_pap_service_name)
 345
 346         # Read policy json from file
 347         with open(policy_template_file) as f:
 348             try:
 349                 policy_json = json.load(f)
 350             except ValueError:
 351                 self.logger.error(policy_template_file + " doesn't seem to contain valid JSON data")
 352                 sys.exit()
 353
 354         # Check policy already applied
 355         requests.packages.urllib3.disable_warnings()
 356         policy_exists_req = requests.get(self.policy_pap_get_url.format(
 357                             p_pap_cluster_ip), auth=self.policy_userpass,
 358                             verify=False, headers=self.policy_headers)
 359         if policy_exists_req.status_code != 200:
 360             self.logger.error('Failure in checking CL policy existence. '
 361                                'Policy-pap responded with HTTP code {0}'.format(
 362                                policy_exists_req.status_code))
 363             sys.exit()
 364
 365         try:
 366             policy_exists_json = policy_exists_req.json()
 367         except ValueError as e:
 368             self.logger.error('Policy-pap request failed: ' + e.message)
 369             sys.exit()
 370
 371         try:
 372             assert policy_exists_json['groups'][0]['pdpSubgroups'] \
 373                                [1]['policies'][0]['name'] != 'operational.vcpe'
 374         except AssertionError:
 375             self.logger.info('vCPE closed loop policy already exists, not applying')
 376             return
 377         except IndexError:
 378             pass # policy doesn't exist
 379
 380         # Create policy
 381         policy_create_req = requests.post(self.policy_api_url.format(
 382                             p_api_cluster_ip), auth=self.policy_userpass,
 383                             json=policy_json, verify=False,
 384                             headers=self.policy_headers)
 385         # Get the policy id from policy-api response
 386         if policy_create_req.status_code != 200:
 387             self.logger.error('Failed creating policy. Policy-api responded'
 388                               ' with HTTP code {0}'.format(policy_create_req.status_code))
 389             sys.exit()
 390
 391         try:
 392             policy_version = json.loads(policy_create_req.text)['policy-version']
 393         except (KeyError, ValueError):
 394             self.logger.error('Policy API response not understood:')
 395             self.logger.debug('\n' + str(policy_create_req.text))
 396
 397         # Inject the policy into Policy PAP
 398         self.policy_pap_json['policies'].append({'policy-version': policy_version})
 399         policy_insert_req = requests.post(self.policy_pap_post_url.format(
 400                             p_pap_cluster_ip), auth=self.policy_userpass,
 401                             json=self.policy_pap_json, verify=False,
 402                             headers=self.policy_headers)
 403         if policy_insert_req.status_code != 200:
 404             self.logger.error('Policy PAP request failed with HTTP code'
 405                               '{0}'.format(policy_insert_req.status_code))
 406             sys.exit()
 407         self.logger.info('Successully pushed closed loop Policy')
 408
 409     def is_node_in_aai(self, node_type, node_uuid):
 410         key = None
 411         search_node_type = None
 412         if node_type == 'service':
 413             search_node_type = 'service-instance'
 414             key = 'service-instance-id'
 415         elif node_type == 'vnf':
 416             search_node_type = 'generic-vnf'
 417             key = 'vnf-id'
 418         else:
 419             logging.error('Invalid node_type: ' + node_type)
 420             sys.exit()
 421
 422         url = 'https://{0}:{1}/aai/v11/search/nodes-query?search-node-type={2}&filter={3}:EQUALS:{4}'.format(
 423             self.hosts['aai-inst1'], self.aai_query_port, search_node_type, key, node_uuid)
 424
 425         headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'X-FromAppID': 'vCPE-Robot', 'X-TransactionId': 'get_aai_subscr'}
 426         requests.packages.urllib3.disable_warnings()
 427         r = requests.get(url, headers=headers, auth=self.aai_userpass, verify=False)
 428         response = r.json()
 429         self.logger.debug('aai query: ' + url)
 430         self.logger.debug('aai response:\n' + json.dumps(response, indent=4, sort_keys=True))
 431         return 'result-data' in response
 432
 433     @staticmethod
 434     def extract_ip_from_str(net_addr, net_addr_len, sz):
 435         """
 436         :param net_addr:  e.g. 10.5.12.0
 437         :param net_addr_len: e.g. 24
 438         :param sz: a string
 439         :return: the first IP address matching the network, e.g. 10.5.12.3
 440         """
 441         network = ipaddress.ip_network(unicode('{0}/{1}'.format(net_addr, net_addr_len)), strict=False)
 442         ip_list = re.findall(r'[0-9]+(?:\.[0-9]+){3}', sz)
 443         for ip in ip_list:
 444             this_net = ipaddress.ip_network(unicode('{0}/{1}'.format(ip, net_addr_len)), strict=False)
 445             if this_net == network:
 446                 return str(ip)
 447         return None
 448
 449     def get_pod_node_oam_ip(self, pod):
 450         """
 451         :Assuming kubectl is available and configured by default config (~/.kube/config)
 452         :param pod: pod name substring, e.g. 'sdnc-sdnc-0'
 453         :return pod's cluster node oam ip (10.0.0.0/16)
 454         """
 455         ret = None
 456         config.load_kube_config()
 457         api = client.CoreV1Api()
 458         kslogger = logging.getLogger('kubernetes')
 459         kslogger.setLevel(logging.INFO)
 460         res = api.list_pod_for_all_namespaces()
 461         for i in res.items:
 462             if pod in i.metadata.name:
 463                 self.logger.debug("found {0}\t{1}\t{2}".format(i.metadata.name, i.status.host_ip, i.spec.node_name))
 464                 ret = i.status.host_ip
 465                 break
 466
 467         if ret is None:
 468             ret = raw_input("Enter " + self.sdnc_controller_pod + " pod cluster node OAM IP address(10.0.0.0/16): ")
 469         return ret
 470
 471     def get_pod_node_public_ip(self, pod):
 472         """
 473         :Assuming kubectl is available and configured by default config (~/.kube/config)
 474         :param pod: pod name substring, e.g. 'sdnc-sdnc-0'
 475         :return pod's cluster node public ip (i.e. 10.12.0.0/16)
 476         """
 477         ret = None
 478         config.load_kube_config()
 479         api = client.CoreV1Api()
 480         kslogger = logging.getLogger('kubernetes')
 481         kslogger.setLevel(logging.INFO)
 482         res = api.list_pod_for_all_namespaces()
 483         for i in res.items:
 484             if pod in i.metadata.name:
 485                 ret = self.get_vm_public_ip_by_nova(i.spec.node_name)
 486                 self.logger.debug("found node {0} public ip: {1}".format(i.spec.node_name, ret))
 487                 break
 488
 489         if ret is None:
 490             ret = raw_input("Enter " + self.sdnc_controller_pod + " pod cluster node public IP address(i.e. " + self.external_net_addr + "): ")
 491         return ret
 492
 493     def get_vm_public_ip_by_nova(self, vm):
 494         """
 495         This method uses openstack nova api to retrieve vm public ip
 496         :param vm: vm name
 497         :return vm public ip
 498         """
 499         subnet = IPNetwork('{0}/{1}'.format(self.external_net_addr, self.external_net_prefix_len))
 500         nova = openstackclient.Client(2, self.cloud['--os-username'], self.cloud['--os-password'], self.cloud['--os-tenant-id'], self.cloud['--os-auth-url'])
 501         for i in nova.servers.list():
 502             if i.name == vm:
 503                 for k, v in i.networks.items():
 504                     for ip in v:
 505                         if IPAddress(ip) in subnet:
 506                             return ip
 507         return None
 508
 509     def get_vm_ip(self, keywords, net_addr=None, net_addr_len=None):
 510         """
 511         :param keywords: list of keywords to search for vm, e.g. ['bng', 'gmux', 'brg']
 512         :param net_addr: e.g. 10.12.5.0
 513         :param net_addr_len: e.g. 24
 514         :return: dictionary {keyword: ip}
 515         """
 516         if not net_addr:
 517             net_addr = self.external_net_addr
 518
 519         if not net_addr_len:
 520             net_addr_len = self.external_net_prefix_len
 521
 522         param = ' '.join([k + ' ' + v for k, v in self.cloud.items() if 'identity' not in k])
 523         openstackcmd = 'nova ' + param + ' list'
 524         self.logger.debug(openstackcmd)
 525
 526         results = os.popen(openstackcmd).read()
 527         all_vm_ip_dict = self.extract_vm_ip_as_dict(results, net_addr, net_addr_len)
 528         latest_vm_list = self.remove_old_vms(all_vm_ip_dict.keys(), self.cpe_vm_prefix)
 529         latest_vm_ip_dict = {vm: all_vm_ip_dict[vm] for vm in latest_vm_list}
 530         ip_dict = self.select_subset_vm_ip(latest_vm_ip_dict, keywords)
 531         if self.oom_mode:
 532             ip_dict.update(self.get_oom_onap_vm_ip(keywords))
 533
 534         if len(ip_dict) != len(keywords):
 535             self.logger.error('Cannot find all desired IP addresses for %s.', keywords)
 536             self.logger.error(json.dumps(ip_dict, indent=4, sort_keys=True))
 537             self.logger.error('Temporarily continue.. remember to check back vcpecommon.py line: 396')
 538 #            sys.exit()
 539         return ip_dict
 540
 541     def get_oom_onap_vm_ip(self, keywords):
 542         vm_ip = {}
 543         onap_vm_list = set(['sdc', 'so', 'sdnc', 'aai-inst1', 'robot', self.dcae_ves_collector_name])
 544         for vm in keywords:
 545             if vm in onap_vm_list:
 546                 vm_ip[vm] = self.oom_so_sdnc_aai_ip
 547         return vm_ip
 548
 549     def get_k8s_service_cluster_ip(self, service):
 550         """
 551         Returns cluster IP for a given service
 552         :param service: name of the service
 553         :return: cluster ip
 554         """
 555         config.load_kube_config()
 556         api = client.CoreV1Api()
 557         kslogger = logging.getLogger('kubernetes')
 558         kslogger.setLevel(logging.INFO)
 559         try:
 560             resp = api.read_namespaced_service(service, self.onap_namespace)
 561         except client.rest.ApiException as e:
 562             self.logger.error('Error while making k8s API request: ' + e.body)
 563             sys.exit()
 564
 565         return resp.spec.cluster_ip
 566
 567     def extract_vm_ip_as_dict(self, novalist_results, net_addr, net_addr_len):
 568         vm_ip_dict = {}
 569         for line in novalist_results.split('\n'):
 570             fields = line.split('|')
 571             if len(fields) == 8:
 572                 vm_name = fields[2]
 573                 ip_info = fields[-2]
 574                 ip = self.extract_ip_from_str(net_addr, net_addr_len, ip_info)
 575                 vm_ip_dict[vm_name] = ip
 576
 577         return vm_ip_dict
 578
 579     def remove_old_vms(self, vm_list, prefix):
 580         """
 581         For vms with format name_timestamp, only keep the one with the latest timestamp.
 582         E.g.,
 583             zdcpe1cpe01brgemu01_201805222148        (drop this)
 584             zdcpe1cpe01brgemu01_201805222229        (keep this)
 585             zdcpe1cpe01gw01_201805162201
 586         """
 587         new_vm_list = []
 588         same_type_vm_dict = {}
 589         for vm in vm_list:
 590             fields = vm.split('_')
 591             if vm.startswith(prefix) and len(fields) == 2 and len(fields[-1]) == len('201805222148') and fields[-1].isdigit():
 592                 if vm > same_type_vm_dict.get(fields[0], '0'):
 593                     same_type_vm_dict[fields[0]] = vm
 594             else:
 595                 new_vm_list.append(vm)
 596
 597         new_vm_list.extend(same_type_vm_dict.values())
 598         return new_vm_list
 599
 600     def select_subset_vm_ip(self, all_vm_ip_dict, vm_name_keyword_list):
 601         vm_ip_dict = {}
 602         for keyword in vm_name_keyword_list:
 603             for vm, ip in all_vm_ip_dict.items():
 604                 if keyword in vm:
 605                     vm_ip_dict[keyword] = ip
 606                     break
 607         return vm_ip_dict
 608
 609     def del_vgmux_ves_mode(self):
 610         url = self.vpp_ves_url.format(self.hosts['mux']) + '/mode'
 611         r = requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 612         self.logger.debug('%s', r)
 613
 614     def del_vgmux_ves_collector(self):
 615         url = self.vpp_ves_url.format(self.hosts['mux']) + '/config'
 616         r = requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 617         self.logger.debug('%s', r)
 618
 619     def set_vgmux_ves_collector(self ):
 620         url = self.vpp_ves_url.format(self.hosts['mux'])
 621         data = {'config':
 622                     {'server-addr': self.hosts[self.dcae_ves_collector_name],
 623                      'server-port': '30235' if self.oom_mode else '8081',
 624                      'read-interval': '10',
 625                      'is-add':'1'
 626                      }
 627                 }
 628         r = requests.post(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass, json=data)
 629         self.logger.debug('%s', r)
 630
 631     def set_vgmux_packet_loss_rate(self, lossrate, vg_vnf_instance_name):
 632         url = self.vpp_ves_url.format(self.hosts['mux'])
 633         data = {"mode":
 634                     {"working-mode": "demo",
 635                      "base-packet-loss": str(lossrate),
 636                      "source-name": vg_vnf_instance_name
 637                      }
 638                 }
 639         r = requests.post(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass, json=data)
 640         self.logger.debug('%s', r)
 641
 642         # return all the VxLAN interface names of BRG or vGMUX based on the IP address
 643     def get_vxlan_interfaces(self, ip, print_info=False):
 644         url = self.vpp_inf_url.format(ip)
 645         self.logger.debug('url is this: %s', url)
 646         r = requests.get(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 647         data = r.json()['interfaces']['interface']
 648         if print_info:
 649             for inf in data:
 650                 if 'name' in inf and 'type' in inf and inf['type'] == 'v3po:vxlan-tunnel':
 651                     print(json.dumps(inf, indent=4, sort_keys=True))
 652
 653         return [inf['name'] for inf in data if 'name' in inf and 'type' in inf and inf['type'] == 'v3po:vxlan-tunnel']
 654
 655     # delete all VxLAN interfaces of each hosts
 656     def delete_vxlan_interfaces(self, host_dic):
 657         for host, ip in host_dic.items():
 658             deleted = False
 659             self.logger.info('{0}: Getting VxLAN interfaces'.format(host))
 660             inf_list = self.get_vxlan_interfaces(ip)
 661             for inf in inf_list:
 662                 deleted = True
 663                 time.sleep(2)
 664                 self.logger.info("{0}: Deleting VxLAN crossconnect {1}".format(host, inf))
 665                 url = self.vpp_inf_url.format(ip) + '/interface/' + inf + '/v3po:l2'
 666                 requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 667
 668             for inf in inf_list:
 669                 deleted = True
 670                 time.sleep(2)
 671                 self.logger.info("{0}: Deleting VxLAN interface {1}".format(host, inf))
 672                 url = self.vpp_inf_url.format(ip) + '/interface/' + inf
 673                 requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 674
 675             if len(self.get_vxlan_interfaces(ip)) > 0:
 676                 self.logger.error("Error deleting VxLAN from {0}, try to restart the VM, IP is {1}.".format(host, ip))
 677                 return False
 678
 679             if not deleted:
 680                 self.logger.info("{0}: no VxLAN interface found, nothing to delete".format(host))
 681         return True
 682
 683     @staticmethod
 684     def save_object(obj, filepathname):
 685         with open(filepathname, 'wb') as fout:
 686             pickle.dump(obj, fout)
 687
 688     @staticmethod
 689     def load_object(filepathname):
 690         with open(filepathname, 'rb') as fin:
 691             return pickle.load(fin)
 692
 693     @staticmethod
 694     def increase_ip_address_or_vni_in_template(vnf_template_file, vnf_parameter_name_list):
 695         with open(vnf_template_file) as json_input:
 696             json_data = json.load(json_input)
 697             param_list = json_data['VNF-API:input']['VNF-API:vnf-topology-information']['VNF-API:vnf-parameters']
 698             for param in param_list:
 699                 if param['vnf-parameter-name'] in vnf_parameter_name_list:
 700                     ipaddr_or_vni = param['vnf-parameter-value'].split('.')
 701                     number = int(ipaddr_or_vni[-1])
 702                     if 254 == number:
 703                         number = 10
 704                     else:
 705                         number = number + 1
 706                     ipaddr_or_vni[-1] = str(number)
 707                     param['vnf-parameter-value'] = '.'.join(ipaddr_or_vni)
 708
 709         assert json_data is not None
 710         with open(vnf_template_file, 'w') as json_output:
 711             json.dump(json_data, json_output, indent=4, sort_keys=True)
 712
 713     def save_preload_data(self, preload_data):
 714         self.save_object(preload_data, self.preload_dict_file)
 715
 716     def load_preload_data(self):
 717         return self.load_object(self.preload_dict_file)
 718
 719     def save_vgmux_vnf_name(self, vgmux_vnf_name):
 720         self.save_object(vgmux_vnf_name, self.vgmux_vnf_name_file)
 721
 722     def load_vgmux_vnf_name(self):
 723         return self.load_object(self.vgmux_vnf_name_file)
 724