test/vcpe/vcpecommon.py

   1 #!/usr/bin/env python
   2
   3 import json
   4 import logging
   5 import os
   6 import pickle
   7 import re
   8 import sys
   9
  10 import ipaddress
  11 import mysql.connector
  12 import requests
  13 import commands
  14 import time
  15 from novaclient import client as openstackclient
  16 from kubernetes import client, config
  17 from netaddr import IPAddress, IPNetwork
  18
  19 ######################################################################
  20 # Parts which must be updated / cross-checked during each deployment #
  21 # are marked as CHANGEME                                             #
  22 ######################################################################
  23
  24 class VcpeCommon:
  25     #############################################################################################
  26     # Set network prefix of k8s host external address; it's used for pod public IP autodetection
  27     # but can be overriden from user in case of autodetection failure
  28     external_net_addr = '10.12.0.0'
  29     external_net_prefix_len = 16
  30
  31     #############################################################################################
  32     # set the openstack cloud access credentials here
  33     oom_mode = False
  34
  35     ###########################
  36     # set Openstack credentials
  37     # CHANGEME part
  38     cloud = {
  39         '--os-auth-url': 'http://10.12.25.2:5000',
  40         '--os-username': 'kxi',
  41         '--os-user-domain-id': 'default',
  42         '--os-project-domain-id': 'default',
  43         '--os-tenant-id': 'bc43d50ffcb84750bac0c1707a9a765b' if oom_mode else '1e097c6713e74fd7ac8e4295e605ee1e',
  44         '--os-region-name': 'RegionOne',
  45         '--os-password': 'n3JhGMGuDzD8',
  46         '--os-project-domain-name': 'Integration-SB-03' if oom_mode else 'Integration-SB-07',
  47         '--os-identity-api-version': '3'
  48     }
  49
  50     ############################################################################
  51     # set oam and public network which must exist in openstack before deployment
  52     # CHANGEME part
  53     common_preload_config = {
  54         'oam_onap_net': 'oam_network_2No2' if oom_mode else 'oam_onap_lAky',
  55         'oam_onap_subnet': 'oam_network_2No2' if oom_mode else 'oam_onap_lAky',
  56         'public_net': 'external',
  57         'public_net_id': '971040b2-7059-49dc-b220-4fab50cb2ad4'
  58     }
  59
  60     #############################################################################
  61     # Set name of Onap's k8s namespace and sdnc controller pod
  62     # CHANGEME part
  63     onap_namespace = 'onap'
  64     onap_environment = 'dev'
  65     sdnc_controller_pod = '-'.join([onap_environment, 'sdnc-sdnc-0'])
  66
  67     template_variable_symbol = '${'
  68     cpe_vm_prefix = 'zdcpe'
  69
  70     #############################################################################################
  71     # preloading network config
  72     #  key=network role
  73     #  value = [subnet_start_ip, subnet_gateway_ip]
  74     preload_network_config = {
  75         'cpe_public': ['10.2.0.2', '10.2.0.1'],
  76         'cpe_signal': ['10.4.0.2', '10.4.0.1'],
  77         'brg_bng': ['10.3.0.2', '10.3.0.1'],
  78         'bng_mux': ['10.1.0.10', '10.1.0.1'],
  79         'mux_gw': ['10.5.0.10', '10.5.0.1']
  80     }
  81
  82     dcae_ves_collector_name = 'dcae-bootstrap'
  83     global_subscriber_id = 'SDN-ETHERNET-INTERNET'
  84     project_name = 'Project-Demonstration'
  85     owning_entity_id = '520cc603-a3c4-4ec2-9ef4-ca70facd79c0'
  86     owning_entity_name = 'OE-Demonstration1'
  87
  88     def __init__(self, extra_host_names=None):
  89         self.logger = logging.getLogger(__name__)
  90         self.logger.setLevel(logging.DEBUG)
  91         self.logger.info('Initializing configuration')
  92
  93         ##################################################################################################################################
  94         # following param must be updated e.g. from csar file (grep for VfModuleModelInvariantUuid string) before vcpe.py customer call !!
  95         # vgw_VfModuleModelInvariantUuid is in rescust service csar,
  96         # look in service-VcpesvcRescust1118-template.yml for groups vgw module metadata. TODO: read this value automatically
  97         # CHANGEME part
  98         self.vgw_VfModuleModelInvariantUuid = '26d6a718-17b2-4ba8-8691-c44343b2ecd2'
  99
 100         # OOM: this is the address that the brg and bng will nat for sdnc access - 10.0.0.x address of k8 host for sdnc-0 container
 101         self.sdnc_oam_ip = self.get_pod_node_oam_ip(self.sdnc_controller_pod)
 102         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
 103         self.oom_so_sdnc_aai_ip = self.get_pod_node_public_ip(self.sdnc_controller_pod)
 104         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
 105         self.oom_dcae_ves_collector = self.oom_so_sdnc_aai_ip
 106         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
 107         self.mr_ip_addr = self.oom_so_sdnc_aai_ip
 108         self.mr_ip_port = '30227'
 109         self.so_nbi_port = '30277' if self.oom_mode else '8080'
 110         self.sdnc_preloading_port = '30267' if self.oom_mode else '8282'
 111         self.aai_query_port = '30233' if self.oom_mode else '8443'
 112         self.sniro_port = '30288' if self.oom_mode else '8080'
 113
 114         self.host_names = ['sdc', 'so', 'sdnc', 'robot', 'aai-inst1', self.dcae_ves_collector_name]
 115         if extra_host_names:
 116             self.host_names.extend(extra_host_names)
 117         # get IP addresses
 118         self.hosts = self.get_vm_ip(self.host_names, self.external_net_addr, self.external_net_prefix_len)
 119         # this is the keyword used to name vgw stack, must not be used in other stacks
 120         self.vgw_name_keyword = 'base_vcpe_vgw'
 121         # this is the file that will keep the index of last assigned SO name
 122         self.vgw_vfmod_name_index_file= '__var/vgw_vfmod_name_index'
 123         self.svc_instance_uuid_file = '__var/svc_instance_uuid'
 124         self.preload_dict_file = '__var/preload_dict'
 125         self.vgmux_vnf_name_file = '__var/vgmux_vnf_name'
 126         self.product_family_id = 'f9457e8c-4afd-45da-9389-46acd9bf5116'
 127         self.custom_product_family_id = 'a9a77d5a-123e-4ca2-9eb9-0b015d2ee0fb'
 128         self.instance_name_prefix = {
 129             'service': 'vcpe_svc',
 130             'network': 'vcpe_net',
 131             'vnf': 'vcpe_vnf',
 132             'vfmodule': 'vcpe_vfmodule'
 133         }
 134         self.aai_userpass = 'AAI', 'AAI'
 135
 136         ############################################################################################################
 137         # following key is overriding public key from vCPE heat templates, it's important to use correct one in here
 138         # CHANGEME part
 139         self.pub_key = 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKXDgoo3+WOqcUG8/5uUbk81+yczgwC4Y8ywTmuQqbNxlY1oQ0YxdMUqUnhitSXs5S/yRuAVOYHwGg2mCs20oAINrP+mxBI544AMIb9itPjCtgqtE2EWo6MmnFGbHB4Sx3XioE7F4VPsh7japsIwzOjbrQe+Mua1TGQ5d4nfEOQaaglXLLPFfuc7WbhbJbK6Q7rHqZfRcOwAMXgDoBqlyqKeiKwnumddo2RyNT8ljYmvB6buz7KnMinzo7qB0uktVT05FH9Rg0CTWH5norlG5qXgP2aukL0gk1ph8iAt7uYLf1ktp+LJI2gaF6L0/qli9EmVCSLr1uJ38Q8CBflhkh'
 140
 141         self.os_tenant_id = self.cloud['--os-tenant-id']
 142         self.os_region_name = self.cloud['--os-region-name']
 143         self.common_preload_config['pub_key'] = self.pub_key
 144         self.sniro_url = 'http://' + self.hosts['robot'] + ':' + self.sniro_port + '/__admin/mappings'
 145         self.sniro_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 146         self.homing_solution = 'sniro'  # value is either 'sniro' or 'oof'
 147 #        self.homing_solution = 'oof'
 148         self.customer_location_used_by_oof = {
 149             "customerLatitude": "32.897480",
 150             "customerLongitude": "-97.040443",
 151             "customerName": "some_company"
 152         }
 153
 154         #############################################################################################
 155         # SDC urls
 156         self.sdc_be_port = '30204'
 157         self.sdc_be_request_userpass = 'vid', 'Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U'
 158         self.sdc_be_request_headers = {'X-ECOMP-InstanceID': 'VID'}
 159         self.sdc_be_url_prefix = 'https://' + self.hosts['sdc'] + ':' + self.sdc_be_port
 160         self.sdc_service_list_url = self.sdc_be_url_prefix + '/sdc/v1/catalog/services'
 161
 162         self.sdc_fe_port = '30207'
 163         self.sdc_fe_request_userpass = 'beep', 'boop'
 164         self.sdc_fe_request_headers = {'USER_ID': 'demo', 'Content-Type': 'application/json'}
 165         self.sdc_fe_url_prefix = 'https://' + self.hosts['sdc'] + ':' + self.sdc_fe_port
 166         self.sdc_get_category_list_url = self.sdc_fe_url_prefix + '/sdc1/feProxy/rest/v1/categories'
 167         self.sdc_create_allotted_resource_subcategory_url = self.sdc_fe_url_prefix + '/sdc1/feProxy/rest/v1/category/resources/resourceNewCategory.allotted%20resource/subCategory'
 168
 169         #############################################################################################
 170         # SDNC urls
 171         self.sdnc_userpass = 'admin', 'Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U'
 172         self.sdnc_db_name = 'sdnctl'
 173         self.sdnc_db_user = 'sdnctl'
 174         self.sdnc_db_pass = 'gamma'
 175         self.sdnc_db_port = '32774'
 176         self.sdnc_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 177         self.sdnc_preload_network_url = 'https://' + self.hosts['sdnc'] + \
 178                                         ':' + self.sdnc_preloading_port + '/restconf/operations/VNF-API:preload-network-topology-operation'
 179         self.sdnc_preload_vnf_url = 'https://' + self.hosts['sdnc'] + \
 180                                     ':' + self.sdnc_preloading_port + '/restconf/operations/VNF-API:preload-vnf-topology-operation'
 181         self.sdnc_preload_gra_url = 'https://' + self.hosts['sdnc'] + \
 182                                     ':' + self.sdnc_preloading_port + '/restconf/operations/GENERIC-RESOURCE-API:preload-vf-module-topology-operation'
 183         self.sdnc_ar_cleanup_url = 'https://' + self.hosts['sdnc'] + ':' + self.sdnc_preloading_port + \
 184                                    '/restconf/config/GENERIC-RESOURCE-API:'
 185
 186         #############################################################################################
 187         # SO urls, note: do NOT add a '/' at the end of the url
 188         self.so_req_api_url = {'v4': 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/serviceInstantiation/v7/serviceInstances',
 189                            'v5': 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/serviceInstantiation/v7/serviceInstances'}
 190         self.so_check_progress_api_url = 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/orchestrationRequests/v6'
 191         self.so_userpass = 'InfraPortalClient', 'password1$'
 192         self.so_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 193         self.so_db_name = 'catalogdb'
 194         self.so_db_user = 'root'
 195         self.so_db_pass = 'secretpassword'
 196         self.so_db_port = '30252' if self.oom_mode else '32769'
 197
 198         self.vpp_inf_url = 'http://{0}:8183/restconf/config/ietf-interfaces:interfaces'
 199         self.vpp_api_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 200         self.vpp_api_userpass = ('admin', 'admin')
 201         self.vpp_ves_url= 'http://{0}:8183/restconf/config/vesagent:vesagent'
 202
 203         #############################################################################################
 204         # POLICY urls
 205         self.policy_userpass = ('healthcheck', 'zb!XztG34')
 206         self.policy_headers = {'Accept': 'application/json', 'Content-Type': 'application/json'}
 207         self.policy_api_url = 'https://{0}:6969/policy/api/v1/policytypes/onap.policies.controlloop.Operational/versions/1.0.0/policies'
 208         self.policy_pap_get_url = 'https://{0}:6969/policy/pap/v1/pdps'
 209         self.policy_pap_json = {'policies': [{'policy-id': 'operational.vcpe'}]}
 210         self.policy_pap_post_url = self.policy_pap_get_url + '/policies'
 211         self.policy_api_service_name = 'policy-api'
 212         self.policy_pap_service_name = 'policy-pap'
 213
 214         #############################################################################################
 215         # MARIADB-GALERA settings
 216         self.mariadb_galera_endpoint_ip = self.get_k8s_service_endpoint_info('mariadb-galera','ip')
 217         self.mariadb_galera_endpoint_port = self.get_k8s_service_endpoint_info('mariadb-galera','port')
 218
 219     def heatbridge(self, openstack_stack_name, svc_instance_uuid):
 220         """
 221         Add vserver information to AAI
 222         """
 223         self.logger.info('Adding vServer information to AAI for {0}'.format(openstack_stack_name))
 224         if not self.oom_mode:
 225             cmd = '/opt/demo.sh heatbridge {0} {1} vCPE'.format(openstack_stack_name, svc_instance_uuid)
 226             ret = commands.getstatusoutput("ssh -i onap_dev root@{0} '{1}'".format(self.hosts['robot'], cmd))
 227             self.logger.debug('%s', ret)
 228         else:
 229             print('To add vGMUX vserver info to AAI, do the following:')
 230             print('- ssh to rancher')
 231             print('- sudo su -')
 232             print('- cd /root/oom/kubernetes/robot')
 233             print('- ./demo-k8s.sh onap heatbridge {0} {1} vCPE'.format(openstack_stack_name, svc_instance_uuid))
 234
 235     def get_brg_mac_from_sdnc(self):
 236         """
 237         Check table DHCP_MAP in the SDNC DB. Find the newly instantiated BRG MAC address.
 238         Note that there might be multiple BRGs, the most recently instantiated BRG always has the largest IP address.
 239         """
 240         cnx = mysql.connector.connect(user=self.sdnc_db_user, password=self.sdnc_db_pass, database=self.sdnc_db_name,
 241                                       host=self.hosts['sdnc'], port=self.sdnc_db_port)
 242         cursor = cnx.cursor()
 243         query = "SELECT * from DHCP_MAP"
 244         cursor.execute(query)
 245
 246         self.logger.debug('DHCP_MAP table in SDNC')
 247         mac_recent = None
 248         host = -1
 249         for mac, ip in cursor:
 250             self.logger.debug(mac + ':' + ip)
 251             this_host = int(ip.split('.')[-1])
 252             if host < this_host:
 253                 host = this_host
 254                 mac_recent = mac
 255
 256         cnx.close()
 257
 258         assert mac_recent
 259         return mac_recent
 260
 261     def execute_cmds_mariadb(self, cmds):
 262         self.execute_cmds_db(cmds, self.sdnc_db_user, self.sdnc_db_pass,
 263                              self.sdnc_db_name, self.mariadb_galera_endpoint_ip,
 264                              self.mariadb_galera_endpoint_port)
 265
 266     def execute_cmds_sdnc_db(self, cmds):
 267         self.execute_cmds_db(cmds, self.sdnc_db_user, self.sdnc_db_pass, self.sdnc_db_name,
 268                              self.hosts['sdnc'], self.sdnc_db_port)
 269
 270     def execute_cmds_so_db(self, cmds):
 271         self.execute_cmds_db(cmds, self.so_db_user, self.so_db_pass, self.so_db_name,
 272                              self.hosts['so'], self.so_db_port)
 273
 274     def execute_cmds_db(self, cmds, dbuser, dbpass, dbname, host, port):
 275         cnx = mysql.connector.connect(user=dbuser, password=dbpass, database=dbname, host=host, port=port)
 276         cursor = cnx.cursor()
 277         for cmd in cmds:
 278             self.logger.debug(cmd)
 279             cursor.execute(cmd)
 280             self.logger.debug('%s', cursor)
 281         cnx.commit()
 282         cursor.close()
 283         cnx.close()
 284
 285     def find_file(self, file_name_keyword, file_ext, search_dir):
 286         """
 287         :param file_name_keyword:  keyword used to look for the csar file, case insensitive matching, e.g, infra
 288         :param file_ext: e.g., csar, json
 289         :param search_dir path to search
 290         :return: path name of the file
 291         """
 292         file_name_keyword = file_name_keyword.lower()
 293         file_ext = file_ext.lower()
 294         if not file_ext.startswith('.'):
 295             file_ext = '.' + file_ext
 296
 297         filenamepath = None
 298         for file_name in os.listdir(search_dir):
 299             file_name_lower = file_name.lower()
 300             if file_name_keyword in file_name_lower and file_name_lower.endswith(file_ext):
 301                 if filenamepath:
 302                     self.logger.error('Multiple files found for *{0}*.{1} in '
 303                                       'directory {2}'.format(file_name_keyword, file_ext, search_dir))
 304                     sys.exit(1)
 305                 filenamepath = os.path.abspath(os.path.join(search_dir, file_name))
 306
 307         if filenamepath:
 308             return filenamepath
 309         else:
 310             self.logger.error("Cannot find *{0}*{1} in directory {2}".format(file_name_keyword, file_ext, search_dir))
 311             sys.exit(1)
 312
 313     @staticmethod
 314     def network_name_to_subnet_name(network_name):
 315         """
 316         :param network_name: example: vcpe_net_cpe_signal_201711281221
 317         :return: vcpe_net_cpe_signal_subnet_201711281221
 318         """
 319         fields = network_name.split('_')
 320         fields.insert(-1, 'subnet')
 321         return '_'.join(fields)
 322
 323     def set_network_name(self, network_name):
 324         param = ' '.join([k + ' ' + v for k, v in self.cloud.items()])
 325         openstackcmd = 'openstack ' + param
 326         cmd = ' '.join([openstackcmd, 'network set --name', network_name, 'ONAP-NW1'])
 327         os.popen(cmd)
 328
 329     def set_subnet_name(self, network_name):
 330         """
 331         Example: network_name =  vcpe_net_cpe_signal_201711281221
 332         set subnet name to vcpe_net_cpe_signal_subnet_201711281221
 333         :return:
 334         """
 335         param = ' '.join([k + ' ' + v for k, v in self.cloud.items()])
 336         openstackcmd = 'openstack ' + param
 337
 338         # expected results: | subnets | subnet_id |
 339         subnet_info = os.popen(openstackcmd + ' network show ' + network_name + ' |grep subnets').read().split('|')
 340         if len(subnet_info) > 2 and subnet_info[1].strip() == 'subnets':
 341             subnet_id = subnet_info[2].strip()
 342             subnet_name = self.network_name_to_subnet_name(network_name)
 343             cmd = ' '.join([openstackcmd, 'subnet set --name', subnet_name, subnet_id])
 344             os.popen(cmd)
 345             self.logger.info("Subnet name set to: " + subnet_name)
 346             return True
 347         else:
 348             self.logger.error("Can't get subnet info from network name: " + network_name)
 349             return False
 350
 351     def set_closed_loop_policy(self, policy_template_file):
 352         # Gather policy services cluster ips
 353         p_api_cluster_ip = self.get_k8s_service_cluster_ip(self.policy_api_service_name)
 354         p_pap_cluster_ip = self.get_k8s_service_cluster_ip(self.policy_pap_service_name)
 355
 356         # Read policy json from file
 357         with open(policy_template_file) as f:
 358             try:
 359                 policy_json = json.load(f)
 360             except ValueError:
 361                 self.logger.error(policy_template_file + " doesn't seem to contain valid JSON data")
 362                 sys.exit(1)
 363
 364         # Check policy already applied
 365         requests.packages.urllib3.disable_warnings()
 366         policy_exists_req = requests.get(self.policy_pap_get_url.format(
 367                             p_pap_cluster_ip), auth=self.policy_userpass,
 368                             verify=False, headers=self.policy_headers)
 369         if policy_exists_req.status_code != 200:
 370             self.logger.error('Failure in checking CL policy existence. '
 371                                'Policy-pap responded with HTTP code {0}'.format(
 372                                policy_exists_req.status_code))
 373             sys.exit(1)
 374
 375         try:
 376             policy_exists_json = policy_exists_req.json()
 377         except ValueError as e:
 378             self.logger.error('Policy-pap request failed: ' + e.message)
 379             sys.exit(1)
 380
 381         try:
 382             assert policy_exists_json['groups'][0]['pdpSubgroups'] \
 383                                [1]['policies'][0]['name'] != 'operational.vcpe'
 384         except AssertionError:
 385             self.logger.info('vCPE closed loop policy already exists, not applying')
 386             return
 387         except IndexError:
 388             pass # policy doesn't exist
 389
 390         # Create policy
 391         policy_create_req = requests.post(self.policy_api_url.format(
 392                             p_api_cluster_ip), auth=self.policy_userpass,
 393                             json=policy_json, verify=False,
 394                             headers=self.policy_headers)
 395         # Get the policy id from policy-api response
 396         if policy_create_req.status_code != 200:
 397             self.logger.error('Failed creating policy. Policy-api responded'
 398                               ' with HTTP code {0}'.format(policy_create_req.status_code))
 399             sys.exit(1)
 400
 401         try:
 402             policy_version = json.loads(policy_create_req.text)['policy-version']
 403         except (KeyError, ValueError):
 404             self.logger.error('Policy API response not understood:')
 405             self.logger.debug('\n' + str(policy_create_req.text))
 406
 407         # Inject the policy into Policy PAP
 408         self.policy_pap_json['policies'].append({'policy-version': policy_version})
 409         policy_insert_req = requests.post(self.policy_pap_post_url.format(
 410                             p_pap_cluster_ip), auth=self.policy_userpass,
 411                             json=self.policy_pap_json, verify=False,
 412                             headers=self.policy_headers)
 413         if policy_insert_req.status_code != 200:
 414             self.logger.error('Policy PAP request failed with HTTP code'
 415                               '{0}'.format(policy_insert_req.status_code))
 416             sys.exit(1)
 417         self.logger.info('Successully pushed closed loop Policy')
 418
 419     def is_node_in_aai(self, node_type, node_uuid):
 420         key = None
 421         search_node_type = None
 422         if node_type == 'service':
 423             search_node_type = 'service-instance'
 424             key = 'service-instance-id'
 425         elif node_type == 'vnf':
 426             search_node_type = 'generic-vnf'
 427             key = 'vnf-id'
 428         else:
 429             logging.error('Invalid node_type: ' + node_type)
 430             sys.exit(1)
 431
 432         url = 'https://{0}:{1}/aai/v11/search/nodes-query?search-node-type={2}&filter={3}:EQUALS:{4}'.format(
 433             self.hosts['aai-inst1'], self.aai_query_port, search_node_type, key, node_uuid)
 434
 435         headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'X-FromAppID': 'vCPE-Robot', 'X-TransactionId': 'get_aai_subscr'}
 436         requests.packages.urllib3.disable_warnings()
 437         r = requests.get(url, headers=headers, auth=self.aai_userpass, verify=False)
 438         response = r.json()
 439         self.logger.debug('aai query: ' + url)
 440         self.logger.debug('aai response:\n' + json.dumps(response, indent=4, sort_keys=True))
 441         return 'result-data' in response
 442
 443     @staticmethod
 444     def extract_ip_from_str(net_addr, net_addr_len, sz):
 445         """
 446         :param net_addr:  e.g. 10.5.12.0
 447         :param net_addr_len: e.g. 24
 448         :param sz: a string
 449         :return: the first IP address matching the network, e.g. 10.5.12.3
 450         """
 451         network = ipaddress.ip_network(unicode('{0}/{1}'.format(net_addr, net_addr_len)), strict=False)
 452         ip_list = re.findall(r'[0-9]+(?:\.[0-9]+){3}', sz)
 453         for ip in ip_list:
 454             this_net = ipaddress.ip_network(unicode('{0}/{1}'.format(ip, net_addr_len)), strict=False)
 455             if this_net == network:
 456                 return str(ip)
 457         return None
 458
 459     def get_pod_node_oam_ip(self, pod):
 460         """
 461         :Assuming kubectl is available and configured by default config (~/.kube/config)
 462         :param pod: pod name substring, e.g. 'sdnc-sdnc-0'
 463         :return pod's cluster node oam ip (10.0.0.0/16)
 464         """
 465         ret = None
 466         config.load_kube_config()
 467         api = client.CoreV1Api()
 468         kslogger = logging.getLogger('kubernetes')
 469         kslogger.setLevel(logging.INFO)
 470         res = api.list_pod_for_all_namespaces()
 471         for i in res.items:
 472             if pod in i.metadata.name:
 473                 self.logger.debug("found {0}\t{1}\t{2}".format(i.metadata.name, i.status.host_ip, i.spec.node_name))
 474                 ret = i.status.host_ip
 475                 break
 476
 477         if ret is None:
 478             ret = raw_input("Enter " + self.sdnc_controller_pod + " pod cluster node OAM IP address(10.0.0.0/16): ")
 479         return ret
 480
 481     def get_pod_node_public_ip(self, pod):
 482         """
 483         :Assuming kubectl is available and configured by default config (~/.kube/config)
 484         :param pod: pod name substring, e.g. 'sdnc-sdnc-0'
 485         :return pod's cluster node public ip (i.e. 10.12.0.0/16)
 486         """
 487         ret = None
 488         config.load_kube_config()
 489         api = client.CoreV1Api()
 490         kslogger = logging.getLogger('kubernetes')
 491         kslogger.setLevel(logging.INFO)
 492         res = api.list_pod_for_all_namespaces()
 493         for i in res.items:
 494             if pod in i.metadata.name:
 495                 ret = self.get_vm_public_ip_by_nova(i.spec.node_name)
 496                 self.logger.debug("found node {0} public ip: {1}".format(i.spec.node_name, ret))
 497                 break
 498
 499         if ret is None:
 500             ret = raw_input("Enter " + self.sdnc_controller_pod + " pod cluster node public IP address(i.e. " + self.external_net_addr + "): ")
 501         return ret
 502
 503     def get_vm_public_ip_by_nova(self, vm):
 504         """
 505         This method uses openstack nova api to retrieve vm public ip
 506         :param vm: vm name
 507         :return vm public ip
 508         """
 509         subnet = IPNetwork('{0}/{1}'.format(self.external_net_addr, self.external_net_prefix_len))
 510         nova = openstackclient.Client(2, self.cloud['--os-username'], self.cloud['--os-password'], self.cloud['--os-tenant-id'], self.cloud['--os-auth-url'])
 511         for i in nova.servers.list():
 512             if i.name == vm:
 513                 for k, v in i.networks.items():
 514                     for ip in v:
 515                         if IPAddress(ip) in subnet:
 516                             return ip
 517         return None
 518
 519     def get_vm_ip(self, keywords, net_addr=None, net_addr_len=None):
 520         """
 521         :param keywords: list of keywords to search for vm, e.g. ['bng', 'gmux', 'brg']
 522         :param net_addr: e.g. 10.12.5.0
 523         :param net_addr_len: e.g. 24
 524         :return: dictionary {keyword: ip}
 525         """
 526         if not net_addr:
 527             net_addr = self.external_net_addr
 528
 529         if not net_addr_len:
 530             net_addr_len = self.external_net_prefix_len
 531
 532         param = ' '.join([k + ' ' + v for k, v in self.cloud.items() if 'identity' not in k])
 533         openstackcmd = 'nova ' + param + ' list'
 534         self.logger.debug(openstackcmd)
 535
 536         results = os.popen(openstackcmd).read()
 537         all_vm_ip_dict = self.extract_vm_ip_as_dict(results, net_addr, net_addr_len)
 538         latest_vm_list = self.remove_old_vms(all_vm_ip_dict.keys(), self.cpe_vm_prefix)
 539         latest_vm_ip_dict = {vm: all_vm_ip_dict[vm] for vm in latest_vm_list}
 540         ip_dict = self.select_subset_vm_ip(latest_vm_ip_dict, keywords)
 541         if self.oom_mode:
 542             ip_dict.update(self.get_oom_onap_vm_ip(keywords))
 543
 544         if len(ip_dict) != len(keywords):
 545             self.logger.error('Cannot find all desired IP addresses for %s.', keywords)
 546             self.logger.error(json.dumps(ip_dict, indent=4, sort_keys=True))
 547             self.logger.error('Temporarily continue.. remember to check back vcpecommon.py line: 396')
 548 #            sys.exit(1)
 549         return ip_dict
 550
 551     def get_oom_onap_vm_ip(self, keywords):
 552         vm_ip = {}
 553         for vm in keywords:
 554             if vm in self.host_names:
 555                 vm_ip[vm] = self.oom_so_sdnc_aai_ip
 556         return vm_ip
 557
 558     def get_k8s_service_cluster_ip(self, service):
 559         """
 560         Returns cluster IP for a given service
 561         :param service: name of the service
 562         :return: cluster ip
 563         """
 564         config.load_kube_config()
 565         api = client.CoreV1Api()
 566         kslogger = logging.getLogger('kubernetes')
 567         kslogger.setLevel(logging.INFO)
 568         try:
 569             resp = api.read_namespaced_service(service, self.onap_namespace)
 570         except client.rest.ApiException as e:
 571             self.logger.error('Error while making k8s API request: ' + e.body)
 572             sys.exit(1)
 573
 574         return resp.spec.cluster_ip
 575
 576     def get_k8s_service_endpoint_info(self, service, subset):
 577         """
 578         Returns endpoint data for a given service and subset. If there
 579         is more than one endpoint returns data for the first one from
 580         the list that API returned.
 581         :param service: name of the service
 582         :param subset: subset name, one of "ip","port"
 583         :return: endpoint ip
 584         """
 585         config.load_kube_config()
 586         api = client.CoreV1Api()
 587         kslogger = logging.getLogger('kubernetes')
 588         kslogger.setLevel(logging.INFO)
 589         try:
 590             resp = api.read_namespaced_endpoints(service, self.onap_namespace)
 591         except client.rest.ApiException as e:
 592             self.logger.error('Error while making k8s API request: ' + e.body)
 593             sys.exit(1)
 594
 595         if subset == "ip":
 596             return resp.subsets[0].addresses[0].ip
 597         elif subset == "port":
 598             return resp.subsets[0].ports[0].port
 599         else:
 600             self.logger.error("Unsupported subset type")
 601
 602     def extract_vm_ip_as_dict(self, novalist_results, net_addr, net_addr_len):
 603         vm_ip_dict = {}
 604         for line in novalist_results.split('\n'):
 605             fields = line.split('|')
 606             if len(fields) == 8:
 607                 vm_name = fields[2]
 608                 ip_info = fields[-2]
 609                 ip = self.extract_ip_from_str(net_addr, net_addr_len, ip_info)
 610                 vm_ip_dict[vm_name] = ip
 611
 612         return vm_ip_dict
 613
 614     def remove_old_vms(self, vm_list, prefix):
 615         """
 616         For vms with format name_timestamp, only keep the one with the latest timestamp.
 617         E.g.,
 618             zdcpe1cpe01brgemu01_201805222148        (drop this)
 619             zdcpe1cpe01brgemu01_201805222229        (keep this)
 620             zdcpe1cpe01gw01_201805162201
 621         """
 622         new_vm_list = []
 623         same_type_vm_dict = {}
 624         for vm in vm_list:
 625             fields = vm.split('_')
 626             if vm.startswith(prefix) and len(fields) == 2 and len(fields[-1]) == len('201805222148') and fields[-1].isdigit():
 627                 if vm > same_type_vm_dict.get(fields[0], '0'):
 628                     same_type_vm_dict[fields[0]] = vm
 629             else:
 630                 new_vm_list.append(vm)
 631
 632         new_vm_list.extend(same_type_vm_dict.values())
 633         return new_vm_list
 634
 635     def select_subset_vm_ip(self, all_vm_ip_dict, vm_name_keyword_list):
 636         vm_ip_dict = {}
 637         for keyword in vm_name_keyword_list:
 638             for vm, ip in all_vm_ip_dict.items():
 639                 if keyword in vm:
 640                     vm_ip_dict[keyword] = ip
 641                     break
 642         return vm_ip_dict
 643
 644     def del_vgmux_ves_mode(self):
 645         url = self.vpp_ves_url.format(self.hosts['mux']) + '/mode'
 646         r = requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 647         self.logger.debug('%s', r)
 648
 649     def del_vgmux_ves_collector(self):
 650         url = self.vpp_ves_url.format(self.hosts['mux']) + '/config'
 651         r = requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 652         self.logger.debug('%s', r)
 653
 654     def set_vgmux_ves_collector(self ):
 655         url = self.vpp_ves_url.format(self.hosts['mux'])
 656         data = {'config':
 657                     {'server-addr': self.hosts[self.dcae_ves_collector_name],
 658                      'server-port': '30235' if self.oom_mode else '8081',
 659                      'read-interval': '10',
 660                      'is-add':'1'
 661                      }
 662                 }
 663         r = requests.post(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass, json=data)
 664         self.logger.debug('%s', r)
 665
 666     def set_vgmux_packet_loss_rate(self, lossrate, vg_vnf_instance_name):
 667         url = self.vpp_ves_url.format(self.hosts['mux'])
 668         data = {"mode":
 669                     {"working-mode": "demo",
 670                      "base-packet-loss": str(lossrate),
 671                      "source-name": vg_vnf_instance_name
 672                      }
 673                 }
 674         r = requests.post(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass, json=data)
 675         self.logger.debug('%s', r)
 676
 677         # return all the VxLAN interface names of BRG or vGMUX based on the IP address
 678     def get_vxlan_interfaces(self, ip, print_info=False):
 679         url = self.vpp_inf_url.format(ip)
 680         self.logger.debug('url is this: %s', url)
 681         r = requests.get(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 682         data = r.json()['interfaces']['interface']
 683         if print_info:
 684             for inf in data:
 685                 if 'name' in inf and 'type' in inf and inf['type'] == 'v3po:vxlan-tunnel':
 686                     print(json.dumps(inf, indent=4, sort_keys=True))
 687
 688         return [inf['name'] for inf in data if 'name' in inf and 'type' in inf and inf['type'] == 'v3po:vxlan-tunnel']
 689
 690     # delete all VxLAN interfaces of each hosts
 691     def delete_vxlan_interfaces(self, host_dic):
 692         for host, ip in host_dic.items():
 693             deleted = False
 694             self.logger.info('{0}: Getting VxLAN interfaces'.format(host))
 695             inf_list = self.get_vxlan_interfaces(ip)
 696             for inf in inf_list:
 697                 deleted = True
 698                 time.sleep(2)
 699                 self.logger.info("{0}: Deleting VxLAN crossconnect {1}".format(host, inf))
 700                 url = self.vpp_inf_url.format(ip) + '/interface/' + inf + '/v3po:l2'
 701                 requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 702
 703             for inf in inf_list:
 704                 deleted = True
 705                 time.sleep(2)
 706                 self.logger.info("{0}: Deleting VxLAN interface {1}".format(host, inf))
 707                 url = self.vpp_inf_url.format(ip) + '/interface/' + inf
 708                 requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 709
 710             if len(self.get_vxlan_interfaces(ip)) > 0:
 711                 self.logger.error("Error deleting VxLAN from {0}, try to restart the VM, IP is {1}.".format(host, ip))
 712                 return False
 713
 714             if not deleted:
 715                 self.logger.info("{0}: no VxLAN interface found, nothing to delete".format(host))
 716         return True
 717
 718     @staticmethod
 719     def save_object(obj, filepathname):
 720         with open(filepathname, 'wb') as fout:
 721             pickle.dump(obj, fout)
 722
 723     @staticmethod
 724     def load_object(filepathname):
 725         with open(filepathname, 'rb') as fin:
 726             return pickle.load(fin)
 727
 728     @staticmethod
 729     def increase_ip_address_or_vni_in_template(vnf_template_file, vnf_parameter_name_list):
 730         with open(vnf_template_file) as json_input:
 731             json_data = json.load(json_input)
 732             param_list = json_data['VNF-API:input']['VNF-API:vnf-topology-information']['VNF-API:vnf-parameters']
 733             for param in param_list:
 734                 if param['vnf-parameter-name'] in vnf_parameter_name_list:
 735                     ipaddr_or_vni = param['vnf-parameter-value'].split('.')
 736                     number = int(ipaddr_or_vni[-1])
 737                     if 254 == number:
 738                         number = 10
 739                     else:
 740                         number = number + 1
 741                     ipaddr_or_vni[-1] = str(number)
 742                     param['vnf-parameter-value'] = '.'.join(ipaddr_or_vni)
 743
 744         assert json_data is not None
 745         with open(vnf_template_file, 'w') as json_output:
 746             json.dump(json_data, json_output, indent=4, sort_keys=True)
 747
 748     def save_preload_data(self, preload_data):
 749         self.save_object(preload_data, self.preload_dict_file)
 750
 751     def load_preload_data(self):
 752         return self.load_object(self.preload_dict_file)
 753
 754     def save_vgmux_vnf_name(self, vgmux_vnf_name):
 755         self.save_object(vgmux_vnf_name, self.vgmux_vnf_name_file)
 756
 757     def load_vgmux_vnf_name(self):
 758         return self.load_object(self.vgmux_vnf_name_file)
 759