test/vcpe/vcpecommon.py

   1 #!/usr/bin/env python
   2
   3 import json
   4 import logging
   5 import os
   6 import pickle
   7 import re
   8 import sys
   9
  10 import ipaddress
  11 import mysql.connector
  12 import requests
  13 import commands
  14 import time
  15 from novaclient import client as openstackclient
  16 from kubernetes import client, config
  17 from netaddr import IPAddress, IPNetwork
  18
  19 ######################################################################
  20 # Parts which must be updated / cross-checked during each deployment #
  21 # are marked as CHANGEME                                             #
  22 ######################################################################
  23
  24 class VcpeCommon:
  25     #############################################################################################
  26     # Set network prefix of k8s host external address; it's used for pod public IP autodetection
  27     # but can be overriden from user in case of autodetection failure
  28     external_net_addr = '10.12.0.0'
  29     external_net_prefix_len = 16
  30
  31     #############################################################################################
  32     # set the openstack cloud access credentials here
  33     oom_mode = True
  34
  35     ###########################
  36     # set Openstack credentials
  37     # CHANGEME part
  38     cloud = {
  39         '--os-auth-url': 'http://10.12.25.2:5000',
  40         '--os-username': 'kxi',
  41         '--os-user-domain-id': 'default',
  42         '--os-project-domain-id': 'default',
  43         '--os-tenant-id': 'bc43d50ffcb84750bac0c1707a9a765b' if oom_mode else '1e097c6713e74fd7ac8e4295e605ee1e',
  44         '--os-region-name': 'RegionOne',
  45         '--os-password': 'n3JhGMGuDzD8',
  46         '--os-project-domain-name': 'Integration-SB-03' if oom_mode else 'Integration-SB-07',
  47         '--os-identity-api-version': '3'
  48     }
  49
  50     ############################################################################
  51     # set oam and public network which must exist in openstack before deployment
  52     # CHANGEME part
  53     common_preload_config = {
  54         'oam_onap_net': 'oam_network_2No2' if oom_mode else 'oam_onap_lAky',
  55         'oam_onap_subnet': 'oam_network_2No2' if oom_mode else 'oam_onap_lAky',
  56         'public_net': 'external',
  57         'public_net_id': '971040b2-7059-49dc-b220-4fab50cb2ad4'
  58     }
  59
  60     #############################################################################
  61     # Set name of Onap's k8s namespace and sdnc controller pod
  62     # CHANGEME part
  63     onap_namespace = 'onap'
  64     onap_environment = 'dev'
  65     sdnc_controller_pod = '-'.join([onap_environment, 'sdnc-sdnc-0'])
  66
  67     template_variable_symbol = '${'
  68     cpe_vm_prefix = 'zdcpe'
  69
  70     #############################################################################################
  71     # preloading network config
  72     #  key=network role
  73     #  value = [subnet_start_ip, subnet_gateway_ip]
  74     preload_network_config = {
  75         'cpe_public': ['10.2.0.2', '10.2.0.1'],
  76         'cpe_signal': ['10.4.0.2', '10.4.0.1'],
  77         'brg_bng': ['10.3.0.2', '10.3.0.1'],
  78         'bng_mux': ['10.1.0.10', '10.1.0.1'],
  79         'mux_gw': ['10.5.0.10', '10.5.0.1']
  80     }
  81
  82     dcae_ves_collector_name = 'dcae-bootstrap'
  83     global_subscriber_id = 'SDN-ETHERNET-INTERNET'
  84     project_name = 'Project-Demonstration'
  85     owning_entity_id = '520cc603-a3c4-4ec2-9ef4-ca70facd79c0'
  86     owning_entity_name = 'OE-Demonstration1'
  87
  88     def __init__(self, extra_host_names=None):
  89         self.logger = logging.getLogger(__name__)
  90         self.logger.setLevel(logging.DEBUG)
  91         self.logger.info('Initializing configuration')
  92
  93         ##################################################################################################################################
  94         # following param must be updated e.g. from csar file (grep for VfModuleModelInvariantUuid string) before vcpe.py customer call !!
  95         # vgw_VfModuleModelInvariantUuid is in rescust service csar,
  96         # look in service-VcpesvcRescust1118-template.yml for groups vgw module metadata. TODO: read this value automatically
  97         # CHANGEME part
  98         self.vgw_VfModuleModelInvariantUuid = '26d6a718-17b2-4ba8-8691-c44343b2ecd2'
  99
 100         # OOM: this is the address that the brg and bng will nat for sdnc access - 10.0.0.x address of k8 host for sdnc-0 container
 101         self.sdnc_oam_ip = self.get_pod_node_oam_ip(self.sdnc_controller_pod)
 102         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
 103         self.oom_so_sdnc_aai_ip = self.get_pod_node_public_ip(self.sdnc_controller_pod)
 104         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
 105         self.oom_dcae_ves_collector = self.oom_so_sdnc_aai_ip
 106         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
 107         self.mr_ip_addr = self.oom_so_sdnc_aai_ip
 108         self.mr_ip_port = '30227'
 109         self.so_nbi_port = '30277' if self.oom_mode else '8080'
 110         self.sdnc_preloading_port = '30267' if self.oom_mode else '8282'
 111         self.aai_query_port = '30233' if self.oom_mode else '8443'
 112         self.sniro_port = '30288' if self.oom_mode else '8080'
 113
 114         self.host_names = ['sdc', 'so', 'sdnc', 'robot', 'aai-inst1', self.dcae_ves_collector_name]
 115         if extra_host_names:
 116             self.host_names.extend(extra_host_names)
 117         # get IP addresses
 118         self.hosts = self.get_vm_ip(self.host_names, self.external_net_addr, self.external_net_prefix_len)
 119         # this is the keyword used to name vgw stack, must not be used in other stacks
 120         self.vgw_name_keyword = 'base_vcpe_vgw'
 121         # this is the file that will keep the index of last assigned SO name
 122         self.vgw_vfmod_name_index_file= '__var/vgw_vfmod_name_index'
 123         self.svc_instance_uuid_file = '__var/svc_instance_uuid'
 124         self.preload_dict_file = '__var/preload_dict'
 125         self.vgmux_vnf_name_file = '__var/vgmux_vnf_name'
 126         self.product_family_id = 'f9457e8c-4afd-45da-9389-46acd9bf5116'
 127         self.custom_product_family_id = 'a9a77d5a-123e-4ca2-9eb9-0b015d2ee0fb'
 128         self.instance_name_prefix = {
 129             'service': 'vcpe_svc',
 130             'network': 'vcpe_net',
 131             'vnf': 'vcpe_vnf',
 132             'vfmodule': 'vcpe_vfmodule'
 133         }
 134         self.aai_userpass = 'AAI', 'AAI'
 135
 136         ############################################################################################################
 137         # following key is overriding public key from vCPE heat templates, it's important to use correct one in here
 138         # CHANGEME part
 139         self.pub_key = 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKXDgoo3+WOqcUG8/5uUbk81+yczgwC4Y8ywTmuQqbNxlY1oQ0YxdMUqUnhitSXs5S/yRuAVOYHwGg2mCs20oAINrP+mxBI544AMIb9itPjCtgqtE2EWo6MmnFGbHB4Sx3XioE7F4VPsh7japsIwzOjbrQe+Mua1TGQ5d4nfEOQaaglXLLPFfuc7WbhbJbK6Q7rHqZfRcOwAMXgDoBqlyqKeiKwnumddo2RyNT8ljYmvB6buz7KnMinzo7qB0uktVT05FH9Rg0CTWH5norlG5qXgP2aukL0gk1ph8iAt7uYLf1ktp+LJI2gaF6L0/qli9EmVCSLr1uJ38Q8CBflhkh'
 140
 141         self.os_tenant_id = self.cloud['--os-tenant-id']
 142         self.os_region_name = self.cloud['--os-region-name']
 143         self.common_preload_config['pub_key'] = self.pub_key
 144         self.sniro_url = 'http://' + self.hosts['robot'] + ':' + self.sniro_port + '/__admin/mappings'
 145         self.sniro_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 146         self.homing_solution = 'sniro'  # value is either 'sniro' or 'oof'
 147 #        self.homing_solution = 'oof'
 148         self.customer_location_used_by_oof = {
 149             "customerLatitude": "32.897480",
 150             "customerLongitude": "-97.040443",
 151             "customerName": "some_company"
 152         }
 153
 154         #############################################################################################
 155         # SDC urls
 156         self.sdc_be_port = '30204'
 157         self.sdc_be_request_userpass = 'vid', 'Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U'
 158         self.sdc_be_request_headers = {'X-ECOMP-InstanceID': 'VID'}
 159         self.sdc_be_url_prefix = 'https://' + self.hosts['sdc'] + ':' + self.sdc_be_port
 160         self.sdc_service_list_url = self.sdc_be_url_prefix + '/sdc/v1/catalog/services'
 161
 162         self.sdc_fe_port = '30207'
 163         self.sdc_fe_request_userpass = 'beep', 'boop'
 164         self.sdc_fe_request_headers = {'USER_ID': 'demo', 'Content-Type': 'application/json'}
 165         self.sdc_fe_url_prefix = 'https://' + self.hosts['sdc'] + ':' + self.sdc_fe_port
 166         self.sdc_get_category_list_url = self.sdc_fe_url_prefix + '/sdc1/feProxy/rest/v1/categories'
 167         self.sdc_create_allotted_resource_subcategory_url = self.sdc_fe_url_prefix + '/sdc1/feProxy/rest/v1/category/resources/resourceNewCategory.allotted%20resource/subCategory'
 168
 169         #############################################################################################
 170         # SDNC urls
 171         self.sdnc_userpass = 'admin', 'Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U'
 172         self.sdnc_db_name = 'sdnctl'
 173         self.sdnc_db_user = 'sdnctl'
 174         self.sdnc_db_pass = 'gamma'
 175         self.sdnc_db_port = '32774'
 176         self.sdnc_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 177         self.sdnc_preload_network_url = 'https://' + self.hosts['sdnc'] + \
 178                                         ':' + self.sdnc_preloading_port + '/restconf/operations/VNF-API:preload-network-topology-operation'
 179         self.sdnc_preload_vnf_url = 'https://' + self.hosts['sdnc'] + \
 180                                     ':' + self.sdnc_preloading_port + '/restconf/operations/VNF-API:preload-vnf-topology-operation'
 181         self.sdnc_preload_gra_url = 'https://' + self.hosts['sdnc'] + \
 182                                     ':' + self.sdnc_preloading_port + '/restconf/operations/GENERIC-RESOURCE-API:preload-vf-module-topology-operation'
 183         self.sdnc_ar_cleanup_url = 'https://' + self.hosts['sdnc'] + ':' + self.sdnc_preloading_port + \
 184                                    '/restconf/config/GENERIC-RESOURCE-API:'
 185
 186         #############################################################################################
 187         # SO urls, note: do NOT add a '/' at the end of the url
 188         self.so_req_api_url = {'v4': 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/serviceInstantiation/v7/serviceInstances',
 189                            'v5': 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/serviceInstantiation/v7/serviceInstances'}
 190         self.so_check_progress_api_url = 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/orchestrationRequests/v6'
 191         self.so_userpass = 'InfraPortalClient', 'password1$'
 192         self.so_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 193         self.so_db_name = 'catalogdb'
 194         self.so_db_user = 'root'
 195         self.so_db_pass = 'secretpassword'
 196         self.so_db_port = '30252' if self.oom_mode else '32769'
 197
 198         self.vpp_inf_url = 'http://{0}:8183/restconf/config/ietf-interfaces:interfaces'
 199         self.vpp_api_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 200         self.vpp_api_userpass = ('admin', 'admin')
 201         self.vpp_ves_url= 'http://{0}:8183/restconf/config/vesagent:vesagent'
 202
 203         #############################################################################################
 204         # POLICY urls
 205         self.policy_userpass = ('healthcheck', 'zb!XztG34')
 206         self.policy_headers = {'Accept': 'application/json', 'Content-Type': 'application/json'}
 207         self.policy_api_url = 'https://{0}:6969/policy/api/v1/policytypes/onap.policies.controlloop.Operational/versions/1.0.0/policies'
 208         self.policy_pap_get_url = 'https://{0}:6969/policy/pap/v1/pdps'
 209         self.policy_pap_json = {'policies': [{'policy-id': 'operational.vcpe'}]}
 210         self.policy_pap_post_url = self.policy_pap_get_url + '/policies'
 211         self.policy_api_service_name = 'policy-api'
 212         self.policy_pap_service_name = 'policy-pap'
 213
 214         #############################################################################################
 215         # MARIADB-GALERA settings
 216         self.mariadb_galera_endpoint_ip = self.get_k8s_service_endpoint_info('mariadb-galera','ip')
 217         self.mariadb_galera_endpoint_port = self.get_k8s_service_endpoint_info('mariadb-galera','port')
 218
 219     def heatbridge(self, openstack_stack_name, svc_instance_uuid):
 220         """
 221         Add vserver information to AAI
 222         """
 223         self.logger.info('Adding vServer information to AAI for {0}'.format(openstack_stack_name))
 224         if not self.oom_mode:
 225             cmd = '/opt/demo.sh heatbridge {0} {1} vCPE'.format(openstack_stack_name, svc_instance_uuid)
 226             ret = commands.getstatusoutput("ssh -i onap_dev root@{0} '{1}'".format(self.hosts['robot'], cmd))
 227             self.logger.debug('%s', ret)
 228         else:
 229             print('To add vGMUX vserver info to AAI, do the following:')
 230             print('- ssh to rancher')
 231             print('- sudo su -')
 232             print('- cd /root/oom/kubernetes/robot')
 233             print('- ./demo-k8s.sh onap heatbridge {0} {1} vCPE'.format(openstack_stack_name, svc_instance_uuid))
 234
 235     def get_brg_mac_from_sdnc(self):
 236         """
 237         Check table DHCP_MAP in the SDNC DB. Find the newly instantiated BRG MAC address.
 238         Note that there might be multiple BRGs, the most recently instantiated BRG always has the largest IP address.
 239         """
 240         cnx = mysql.connector.connect(user=self.sdnc_db_user, password=self.sdnc_db_pass, database=self.sdnc_db_name,
 241                                       host=self.hosts['sdnc'], port=self.sdnc_db_port)
 242         cursor = cnx.cursor()
 243         query = "SELECT * from DHCP_MAP"
 244         cursor.execute(query)
 245
 246         self.logger.debug('DHCP_MAP table in SDNC')
 247         mac_recent = None
 248         host = -1
 249         for mac, ip in cursor:
 250             self.logger.debug(mac + ':' + ip)
 251             this_host = int(ip.split('.')[-1])
 252             if host < this_host:
 253                 host = this_host
 254                 mac_recent = mac
 255
 256         cnx.close()
 257
 258         assert mac_recent
 259         return mac_recent
 260
 261     def execute_cmds_mariadb(self, cmds):
 262         self.execute_cmds_db(cmds, self.sdnc_db_user, self.sdnc_db_pass,
 263                              self.sdnc_db_name, self.mariadb_galera_endpoint_ip,
 264                              self.mariadb_galera_endpoint_port)
 265
 266     def execute_cmds_sdnc_db(self, cmds):
 267         self.execute_cmds_db(cmds, self.sdnc_db_user, self.sdnc_db_pass, self.sdnc_db_name,
 268                              self.hosts['sdnc'], self.sdnc_db_port)
 269
 270     def execute_cmds_so_db(self, cmds):
 271         self.execute_cmds_db(cmds, self.so_db_user, self.so_db_pass, self.so_db_name,
 272                              self.hosts['so'], self.so_db_port)
 273
 274     def execute_cmds_db(self, cmds, dbuser, dbpass, dbname, host, port):
 275         cnx = mysql.connector.connect(user=dbuser, password=dbpass, database=dbname, host=host, port=port)
 276         cursor = cnx.cursor()
 277         for cmd in cmds:
 278             self.logger.debug(cmd)
 279             cursor.execute(cmd)
 280             self.logger.debug('%s', cursor)
 281         cnx.commit()
 282         cursor.close()
 283         cnx.close()
 284
 285     def find_file(self, file_name_keyword, file_ext, search_dir):
 286         """
 287         :param file_name_keyword:  keyword used to look for the csar file, case insensitive matching, e.g, infra
 288         :param file_ext: e.g., csar, json
 289         :param search_dir path to search
 290         :return: path name of the file
 291         """
 292         file_name_keyword = file_name_keyword.lower()
 293         file_ext = file_ext.lower()
 294         if not file_ext.startswith('.'):
 295             file_ext = '.' + file_ext
 296
 297         filenamepath = None
 298         for file_name in os.listdir(search_dir):
 299             file_name_lower = file_name.lower()
 300             if file_name_keyword in file_name_lower and file_name_lower.endswith(file_ext):
 301                 if filenamepath:
 302                     self.logger.error('Multiple files found for *{0}*.{1} in '
 303                                       'directory {2}'.format(file_name_keyword, file_ext, search_dir))
 304                     sys.exit()
 305                 filenamepath = os.path.abspath(os.path.join(search_dir, file_name))
 306
 307         if filenamepath:
 308             return filenamepath
 309         else:
 310             self.logger.error("Cannot find *{0}*{1} in directory {2}".format(file_name_keyword, file_ext, search_dir))
 311             sys.exit()
 312
 313     @staticmethod
 314     def network_name_to_subnet_name(network_name):
 315         """
 316         :param network_name: example: vcpe_net_cpe_signal_201711281221
 317         :return: vcpe_net_cpe_signal_subnet_201711281221
 318         """
 319         fields = network_name.split('_')
 320         fields.insert(-1, 'subnet')
 321         return '_'.join(fields)
 322
 323     def set_network_name(self, network_name):
 324         param = ' '.join([k + ' ' + v for k, v in self.cloud.items()])
 325         openstackcmd = 'openstack ' + param
 326         cmd = ' '.join([openstackcmd, 'network set --name', network_name, 'ONAP-NW1'])
 327         os.popen(cmd)
 328
 329     def set_subnet_name(self, network_name):
 330         """
 331         Example: network_name =  vcpe_net_cpe_signal_201711281221
 332         set subnet name to vcpe_net_cpe_signal_subnet_201711281221
 333         :return:
 334         """
 335         param = ' '.join([k + ' ' + v for k, v in self.cloud.items()])
 336         openstackcmd = 'openstack ' + param
 337
 338         # expected results: | subnets | subnet_id |
 339         subnet_info = os.popen(openstackcmd + ' network show ' + network_name + ' |grep subnets').read().split('|')
 340         if len(subnet_info) > 2 and subnet_info[1].strip() == 'subnets':
 341             subnet_id = subnet_info[2].strip()
 342             subnet_name = self.network_name_to_subnet_name(network_name)
 343             cmd = ' '.join([openstackcmd, 'subnet set --name', subnet_name, subnet_id])
 344             os.popen(cmd)
 345             self.logger.info("Subnet name set to: " + subnet_name)
 346             return True
 347         else:
 348             self.logger.error("Can't get subnet info from network name: " + network_name)
 349             return False
 350
 351     def set_closed_loop_policy(self, policy_template_file):
 352         # Gather policy services cluster ips
 353         p_api_cluster_ip = self.get_k8s_service_cluster_ip(self.policy_api_service_name)
 354         p_pap_cluster_ip = self.get_k8s_service_cluster_ip(self.policy_pap_service_name)
 355
 356         # Read policy json from file
 357         with open(policy_template_file) as f:
 358             try:
 359                 policy_json = json.load(f)
 360             except ValueError:
 361                 self.logger.error(policy_template_file + " doesn't seem to contain valid JSON data")
 362                 sys.exit()
 363
 364         # Check policy already applied
 365         requests.packages.urllib3.disable_warnings()
 366         policy_exists_req = requests.get(self.policy_pap_get_url.format(
 367                             p_pap_cluster_ip), auth=self.policy_userpass,
 368                             verify=False, headers=self.policy_headers)
 369         if policy_exists_req.status_code != 200:
 370             self.logger.error('Failure in checking CL policy existence. '
 371                                'Policy-pap responded with HTTP code {0}'.format(
 372                                policy_exists_req.status_code))
 373             sys.exit()
 374
 375         try:
 376             policy_exists_json = policy_exists_req.json()
 377         except ValueError as e:
 378             self.logger.error('Policy-pap request failed: ' + e.message)
 379             sys.exit()
 380
 381         try:
 382             assert policy_exists_json['groups'][0]['pdpSubgroups'] \
 383                                [1]['policies'][0]['name'] != 'operational.vcpe'
 384         except AssertionError:
 385             self.logger.info('vCPE closed loop policy already exists, not applying')
 386             return
 387         except IndexError:
 388             pass # policy doesn't exist
 389
 390         # Create policy
 391         policy_create_req = requests.post(self.policy_api_url.format(
 392                             p_api_cluster_ip), auth=self.policy_userpass,
 393                             json=policy_json, verify=False,
 394                             headers=self.policy_headers)
 395         # Get the policy id from policy-api response
 396         if policy_create_req.status_code != 200:
 397             self.logger.error('Failed creating policy. Policy-api responded'
 398                               ' with HTTP code {0}'.format(policy_create_req.status_code))
 399             sys.exit()
 400
 401         try:
 402             policy_version = json.loads(policy_create_req.text)['policy-version']
 403         except (KeyError, ValueError):
 404             self.logger.error('Policy API response not understood:')
 405             self.logger.debug('\n' + str(policy_create_req.text))
 406
 407         # Inject the policy into Policy PAP
 408         self.policy_pap_json['policies'].append({'policy-version': policy_version})
 409         policy_insert_req = requests.post(self.policy_pap_post_url.format(
 410                             p_pap_cluster_ip), auth=self.policy_userpass,
 411                             json=self.policy_pap_json, verify=False,
 412                             headers=self.policy_headers)
 413         if policy_insert_req.status_code != 200:
 414             self.logger.error('Policy PAP request failed with HTTP code'
 415                               '{0}'.format(policy_insert_req.status_code))
 416             sys.exit()
 417         self.logger.info('Successully pushed closed loop Policy')
 418
 419     def is_node_in_aai(self, node_type, node_uuid):
 420         key = None
 421         search_node_type = None
 422         if node_type == 'service':
 423             search_node_type = 'service-instance'
 424             key = 'service-instance-id'
 425         elif node_type == 'vnf':
 426             search_node_type = 'generic-vnf'
 427             key = 'vnf-id'
 428         else:
 429             logging.error('Invalid node_type: ' + node_type)
 430             sys.exit()
 431
 432         url = 'https://{0}:{1}/aai/v11/search/nodes-query?search-node-type={2}&filter={3}:EQUALS:{4}'.format(
 433             self.hosts['aai-inst1'], self.aai_query_port, search_node_type, key, node_uuid)
 434
 435         headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'X-FromAppID': 'vCPE-Robot', 'X-TransactionId': 'get_aai_subscr'}
 436         requests.packages.urllib3.disable_warnings()
 437         r = requests.get(url, headers=headers, auth=self.aai_userpass, verify=False)
 438         response = r.json()
 439         self.logger.debug('aai query: ' + url)
 440         self.logger.debug('aai response:\n' + json.dumps(response, indent=4, sort_keys=True))
 441         return 'result-data' in response
 442
 443     @staticmethod
 444     def extract_ip_from_str(net_addr, net_addr_len, sz):
 445         """
 446         :param net_addr:  e.g. 10.5.12.0
 447         :param net_addr_len: e.g. 24
 448         :param sz: a string
 449         :return: the first IP address matching the network, e.g. 10.5.12.3
 450         """
 451         network = ipaddress.ip_network(unicode('{0}/{1}'.format(net_addr, net_addr_len)), strict=False)
 452         ip_list = re.findall(r'[0-9]+(?:\.[0-9]+){3}', sz)
 453         for ip in ip_list:
 454             this_net = ipaddress.ip_network(unicode('{0}/{1}'.format(ip, net_addr_len)), strict=False)
 455             if this_net == network:
 456                 return str(ip)
 457         return None
 458
 459     def get_pod_node_oam_ip(self, pod):
 460         """
 461         :Assuming kubectl is available and configured by default config (~/.kube/config)
 462         :param pod: pod name substring, e.g. 'sdnc-sdnc-0'
 463         :return pod's cluster node oam ip (10.0.0.0/16)
 464         """
 465         ret = None
 466         config.load_kube_config()
 467         api = client.CoreV1Api()
 468         kslogger = logging.getLogger('kubernetes')
 469         kslogger.setLevel(logging.INFO)
 470         res = api.list_pod_for_all_namespaces()
 471         for i in res.items:
 472             if pod in i.metadata.name:
 473                 self.logger.debug("found {0}\t{1}\t{2}".format(i.metadata.name, i.status.host_ip, i.spec.node_name))
 474                 ret = i.status.host_ip
 475                 break
 476
 477         if ret is None:
 478             ret = raw_input("Enter " + self.sdnc_controller_pod + " pod cluster node OAM IP address(10.0.0.0/16): ")
 479         return ret
 480
 481     def get_pod_node_public_ip(self, pod):
 482         """
 483         :Assuming kubectl is available and configured by default config (~/.kube/config)
 484         :param pod: pod name substring, e.g. 'sdnc-sdnc-0'
 485         :return pod's cluster node public ip (i.e. 10.12.0.0/16)
 486         """
 487         ret = None
 488         config.load_kube_config()
 489         api = client.CoreV1Api()
 490         kslogger = logging.getLogger('kubernetes')
 491         kslogger.setLevel(logging.INFO)
 492         res = api.list_pod_for_all_namespaces()
 493         for i in res.items:
 494             if pod in i.metadata.name:
 495                 ret = self.get_vm_public_ip_by_nova(i.spec.node_name)
 496                 self.logger.debug("found node {0} public ip: {1}".format(i.spec.node_name, ret))
 497                 break
 498
 499         if ret is None:
 500             ret = raw_input("Enter " + self.sdnc_controller_pod + " pod cluster node public IP address(i.e. " + self.external_net_addr + "): ")
 501         return ret
 502
 503     def get_vm_public_ip_by_nova(self, vm):
 504         """
 505         This method uses openstack nova api to retrieve vm public ip
 506         :param vm: vm name
 507         :return vm public ip
 508         """
 509         subnet = IPNetwork('{0}/{1}'.format(self.external_net_addr, self.external_net_prefix_len))
 510         nova = openstackclient.Client(2, self.cloud['--os-username'], self.cloud['--os-password'], self.cloud['--os-tenant-id'], self.cloud['--os-auth-url'])
 511         for i in nova.servers.list():
 512             if i.name == vm:
 513                 for k, v in i.networks.items():
 514                     for ip in v:
 515                         if IPAddress(ip) in subnet:
 516                             return ip
 517         return None
 518
 519     def get_vm_ip(self, keywords, net_addr=None, net_addr_len=None):
 520         """
 521         :param keywords: list of keywords to search for vm, e.g. ['bng', 'gmux', 'brg']
 522         :param net_addr: e.g. 10.12.5.0
 523         :param net_addr_len: e.g. 24
 524         :return: dictionary {keyword: ip}
 525         """
 526         if not net_addr:
 527             net_addr = self.external_net_addr
 528
 529         if not net_addr_len:
 530             net_addr_len = self.external_net_prefix_len
 531
 532         param = ' '.join([k + ' ' + v for k, v in self.cloud.items() if 'identity' not in k])
 533         openstackcmd = 'nova ' + param + ' list'
 534         self.logger.debug(openstackcmd)
 535
 536         results = os.popen(openstackcmd).read()
 537         all_vm_ip_dict = self.extract_vm_ip_as_dict(results, net_addr, net_addr_len)
 538         latest_vm_list = self.remove_old_vms(all_vm_ip_dict.keys(), self.cpe_vm_prefix)
 539         latest_vm_ip_dict = {vm: all_vm_ip_dict[vm] for vm in latest_vm_list}
 540         ip_dict = self.select_subset_vm_ip(latest_vm_ip_dict, keywords)
 541         if self.oom_mode:
 542             ip_dict.update(self.get_oom_onap_vm_ip(keywords))
 543
 544         if len(ip_dict) != len(keywords):
 545             self.logger.error('Cannot find all desired IP addresses for %s.', keywords)
 546             self.logger.error(json.dumps(ip_dict, indent=4, sort_keys=True))
 547             self.logger.error('Temporarily continue.. remember to check back vcpecommon.py line: 396')
 548 #            sys.exit()
 549         return ip_dict
 550
 551     def get_oom_onap_vm_ip(self, keywords):
 552         vm_ip = {}
 553         onap_vm_list = set(['sdc', 'so', 'sdnc', 'aai-inst1', 'robot', self.dcae_ves_collector_name])
 554         for vm in keywords:
 555             if vm in onap_vm_list:
 556                 vm_ip[vm] = self.oom_so_sdnc_aai_ip
 557         return vm_ip
 558
 559     def get_k8s_service_cluster_ip(self, service):
 560         """
 561         Returns cluster IP for a given service
 562         :param service: name of the service
 563         :return: cluster ip
 564         """
 565         config.load_kube_config()
 566         api = client.CoreV1Api()
 567         kslogger = logging.getLogger('kubernetes')
 568         kslogger.setLevel(logging.INFO)
 569         try:
 570             resp = api.read_namespaced_service(service, self.onap_namespace)
 571         except client.rest.ApiException as e:
 572             self.logger.error('Error while making k8s API request: ' + e.body)
 573             sys.exit()
 574
 575         return resp.spec.cluster_ip
 576
 577     def get_k8s_service_endpoint_info(self, service, subset):
 578         """
 579         Returns endpoint data for a given service and subset. If there
 580         is more than one endpoint returns data for the first one from
 581         the list that API returned.
 582         :param service: name of the service
 583         :param subset: subset name, one of "ip","port"
 584         :return: endpoint ip
 585         """
 586         config.load_kube_config()
 587         api = client.CoreV1Api()
 588         kslogger = logging.getLogger('kubernetes')
 589         kslogger.setLevel(logging.INFO)
 590         try:
 591             resp = api.read_namespaced_endpoints(service, self.onap_namespace)
 592         except client.rest.ApiException as e:
 593             self.logger.error('Error while making k8s API request: ' + e.body)
 594             sys.exit()
 595
 596         if subset == "ip":
 597             return resp.subsets[0].addresses[0].ip
 598         elif subset == "port":
 599             return resp.subsets[0].ports[0].port
 600         else:
 601             self.logger.error("Unsupported subset type")
 602
 603     def extract_vm_ip_as_dict(self, novalist_results, net_addr, net_addr_len):
 604         vm_ip_dict = {}
 605         for line in novalist_results.split('\n'):
 606             fields = line.split('|')
 607             if len(fields) == 8:
 608                 vm_name = fields[2]
 609                 ip_info = fields[-2]
 610                 ip = self.extract_ip_from_str(net_addr, net_addr_len, ip_info)
 611                 vm_ip_dict[vm_name] = ip
 612
 613         return vm_ip_dict
 614
 615     def remove_old_vms(self, vm_list, prefix):
 616         """
 617         For vms with format name_timestamp, only keep the one with the latest timestamp.
 618         E.g.,
 619             zdcpe1cpe01brgemu01_201805222148        (drop this)
 620             zdcpe1cpe01brgemu01_201805222229        (keep this)
 621             zdcpe1cpe01gw01_201805162201
 622         """
 623         new_vm_list = []
 624         same_type_vm_dict = {}
 625         for vm in vm_list:
 626             fields = vm.split('_')
 627             if vm.startswith(prefix) and len(fields) == 2 and len(fields[-1]) == len('201805222148') and fields[-1].isdigit():
 628                 if vm > same_type_vm_dict.get(fields[0], '0'):
 629                     same_type_vm_dict[fields[0]] = vm
 630             else:
 631                 new_vm_list.append(vm)
 632
 633         new_vm_list.extend(same_type_vm_dict.values())
 634         return new_vm_list
 635
 636     def select_subset_vm_ip(self, all_vm_ip_dict, vm_name_keyword_list):
 637         vm_ip_dict = {}
 638         for keyword in vm_name_keyword_list:
 639             for vm, ip in all_vm_ip_dict.items():
 640                 if keyword in vm:
 641                     vm_ip_dict[keyword] = ip
 642                     break
 643         return vm_ip_dict
 644
 645     def del_vgmux_ves_mode(self):
 646         url = self.vpp_ves_url.format(self.hosts['mux']) + '/mode'
 647         r = requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 648         self.logger.debug('%s', r)
 649
 650     def del_vgmux_ves_collector(self):
 651         url = self.vpp_ves_url.format(self.hosts['mux']) + '/config'
 652         r = requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 653         self.logger.debug('%s', r)
 654
 655     def set_vgmux_ves_collector(self ):
 656         url = self.vpp_ves_url.format(self.hosts['mux'])
 657         data = {'config':
 658                     {'server-addr': self.hosts[self.dcae_ves_collector_name],
 659                      'server-port': '30235' if self.oom_mode else '8081',
 660                      'read-interval': '10',
 661                      'is-add':'1'
 662                      }
 663                 }
 664         r = requests.post(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass, json=data)
 665         self.logger.debug('%s', r)
 666
 667     def set_vgmux_packet_loss_rate(self, lossrate, vg_vnf_instance_name):
 668         url = self.vpp_ves_url.format(self.hosts['mux'])
 669         data = {"mode":
 670                     {"working-mode": "demo",
 671                      "base-packet-loss": str(lossrate),
 672                      "source-name": vg_vnf_instance_name
 673                      }
 674                 }
 675         r = requests.post(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass, json=data)
 676         self.logger.debug('%s', r)
 677
 678         # return all the VxLAN interface names of BRG or vGMUX based on the IP address
 679     def get_vxlan_interfaces(self, ip, print_info=False):
 680         url = self.vpp_inf_url.format(ip)
 681         self.logger.debug('url is this: %s', url)
 682         r = requests.get(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 683         data = r.json()['interfaces']['interface']
 684         if print_info:
 685             for inf in data:
 686                 if 'name' in inf and 'type' in inf and inf['type'] == 'v3po:vxlan-tunnel':
 687                     print(json.dumps(inf, indent=4, sort_keys=True))
 688
 689         return [inf['name'] for inf in data if 'name' in inf and 'type' in inf and inf['type'] == 'v3po:vxlan-tunnel']
 690
 691     # delete all VxLAN interfaces of each hosts
 692     def delete_vxlan_interfaces(self, host_dic):
 693         for host, ip in host_dic.items():
 694             deleted = False
 695             self.logger.info('{0}: Getting VxLAN interfaces'.format(host))
 696             inf_list = self.get_vxlan_interfaces(ip)
 697             for inf in inf_list:
 698                 deleted = True
 699                 time.sleep(2)
 700                 self.logger.info("{0}: Deleting VxLAN crossconnect {1}".format(host, inf))
 701                 url = self.vpp_inf_url.format(ip) + '/interface/' + inf + '/v3po:l2'
 702                 requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 703
 704             for inf in inf_list:
 705                 deleted = True
 706                 time.sleep(2)
 707                 self.logger.info("{0}: Deleting VxLAN interface {1}".format(host, inf))
 708                 url = self.vpp_inf_url.format(ip) + '/interface/' + inf
 709                 requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 710
 711             if len(self.get_vxlan_interfaces(ip)) > 0:
 712                 self.logger.error("Error deleting VxLAN from {0}, try to restart the VM, IP is {1}.".format(host, ip))
 713                 return False
 714
 715             if not deleted:
 716                 self.logger.info("{0}: no VxLAN interface found, nothing to delete".format(host))
 717         return True
 718
 719     @staticmethod
 720     def save_object(obj, filepathname):
 721         with open(filepathname, 'wb') as fout:
 722             pickle.dump(obj, fout)
 723
 724     @staticmethod
 725     def load_object(filepathname):
 726         with open(filepathname, 'rb') as fin:
 727             return pickle.load(fin)
 728
 729     @staticmethod
 730     def increase_ip_address_or_vni_in_template(vnf_template_file, vnf_parameter_name_list):
 731         with open(vnf_template_file) as json_input:
 732             json_data = json.load(json_input)
 733             param_list = json_data['VNF-API:input']['VNF-API:vnf-topology-information']['VNF-API:vnf-parameters']
 734             for param in param_list:
 735                 if param['vnf-parameter-name'] in vnf_parameter_name_list:
 736                     ipaddr_or_vni = param['vnf-parameter-value'].split('.')
 737                     number = int(ipaddr_or_vni[-1])
 738                     if 254 == number:
 739                         number = 10
 740                     else:
 741                         number = number + 1
 742                     ipaddr_or_vni[-1] = str(number)
 743                     param['vnf-parameter-value'] = '.'.join(ipaddr_or_vni)
 744
 745         assert json_data is not None
 746         with open(vnf_template_file, 'w') as json_output:
 747             json.dump(json_data, json_output, indent=4, sort_keys=True)
 748
 749     def save_preload_data(self, preload_data):
 750         self.save_object(preload_data, self.preload_dict_file)
 751
 752     def load_preload_data(self):
 753         return self.load_object(self.preload_dict_file)
 754
 755     def save_vgmux_vnf_name(self, vgmux_vnf_name):
 756         self.save_object(vgmux_vnf_name, self.vgmux_vnf_name_file)
 757
 758     def load_vgmux_vnf_name(self):
 759         return self.load_object(self.vgmux_vnf_name_file)
 760