test/vcpe/vcpecommon.py

   1 #!/usr/bin/env python
   2
   3 import json
   4 import logging
   5 import os
   6 import pickle
   7 import re
   8 import sys
   9
  10 import ipaddress
  11 import mysql.connector
  12 import requests
  13 import commands
  14 import time
  15 from novaclient import client as openstackclient
  16 from kubernetes import client, config
  17 from netaddr import IPAddress, IPNetwork
  18
  19 ######################################################################
  20 # Parts which must be updated / cross-checked during each deployment #
  21 # are marked as CHANGEME                                             #
  22 ######################################################################
  23
  24 class VcpeCommon:
  25     #############################################################################################
  26     # Set network prefix of k8s host external address; it's used for pod public IP autodetection
  27     # but can be overriden from user in case of autodetection failure
  28     external_net_addr = '10.12.0.0'
  29     external_net_prefix_len = 16
  30
  31     #############################################################################################
  32     # set the openstack cloud access credentials here
  33     oom_mode = True
  34
  35     #############################################################################################
  36     # set the gra_api flag
  37     #gra_api_flag= False
  38     gra_api_flag= True
  39
  40     ###########################
  41     # set Openstack credentials
  42     # CHANGEME part
  43     cloud = {
  44         '--os-auth-url': 'http://10.12.25.2:5000',
  45         '--os-username': 'kxi',
  46         '--os-user-domain-id': 'default',
  47         '--os-project-domain-id': 'default',
  48         '--os-tenant-id': '712b6016580e410b9abfec9ca34953ce' if oom_mode else '1e097c6713e74fd7ac8e4295e605ee1e',
  49         '--os-region-name': 'RegionOne',
  50         '--os-password': 'n3JhGMGuDzD8',
  51         '--os-project-domain-name': 'Integration-Release-Daily' if oom_mode else 'Integration-SB-07',
  52         '--os-identity-api-version': '3'
  53     }
  54
  55     ############################################################################
  56     # set oam and public network which must exist in openstack before deployment
  57     # CHANGEME part
  58     common_preload_config = {
  59         'oam_onap_net': 'oam_network_exxC' if oom_mode else 'oam_onap_lAky',
  60         'oam_onap_subnet': 'oam_network_exxC' if oom_mode else 'oam_onap_lAky',
  61         'public_net': 'external',
  62         'public_net_id': '971040b2-7059-49dc-b220-4fab50cb2ad4'
  63     }
  64
  65     #############################################################################
  66     # Set name of Onap's k8s namespace and sdnc controller pod
  67     # CHANGEME part
  68     onap_namespace = 'onap'
  69     onap_environment = 'dev'
  70     sdnc_controller_pod = '-'.join([onap_environment, 'sdnc-sdnc-0'])
  71
  72     template_variable_symbol = '${'
  73     cpe_vm_prefix = 'zdcpe'
  74
  75     #############################################################################################
  76     # preloading network config
  77     #  key=network role
  78     #  value = [subnet_start_ip, subnet_gateway_ip]
  79     preload_network_config = {
  80         'cpe_public': ['10.2.0.2', '10.2.0.1'],
  81         'cpe_signal': ['10.4.0.2', '10.4.0.1'],
  82         'brg_bng': ['10.3.0.2', '10.3.0.1'],
  83         'bng_mux': ['10.1.0.10', '10.1.0.1'],
  84         'mux_gw': ['10.5.0.10', '10.5.0.1']
  85     }
  86
  87     dcae_ves_collector_name = 'dcae-bootstrap'
  88     global_subscriber_id = 'SDN-ETHERNET-INTERNET'
  89     project_name = 'Project-Demonstration'
  90     owning_entity_id = '520cc603-a3c4-4ec2-9ef4-ca70facd79c0'
  91     owning_entity_name = 'OE-Demonstration1'
  92
  93     def __init__(self, extra_host_names=None):
  94         self.logger = logging.getLogger(__name__)
  95         self.logger.setLevel(logging.DEBUG)
  96         self.logger.info('Initializing configuration')
  97
  98         ##################################################################################################################################
  99         # following param must be updated e.g. from csar file (grep for VfModuleModelInvariantUuid string) before vcpe.py customer call !!
 100         # vgw_VfModuleModelInvariantUuid is in rescust service csar,
 101         # look in service-VcpesvcRescust1118-template.yml for groups vgw module metadata. TODO: read this value automatically
 102         # CHANGEME part
 103         self.vgw_VfModuleModelInvariantUuid = '26d6a718-17b2-4ba8-8691-c44343b2ecd2'
 104
 105         # OOM: this is the address that the brg and bng will nat for sdnc access - 10.0.0.x address of k8 host for sdnc-0 container
 106         self.sdnc_oam_ip = self.get_pod_node_oam_ip(self.sdnc_controller_pod)
 107         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
 108         self.oom_so_sdnc_aai_ip = self.get_pod_node_public_ip(self.sdnc_controller_pod)
 109         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
 110         self.oom_dcae_ves_collector = self.oom_so_sdnc_aai_ip
 111         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
 112         self.mr_ip_addr = self.oom_so_sdnc_aai_ip
 113         self.mr_ip_port = '30227'
 114         self.so_nbi_port = '30277' if self.oom_mode else '8080'
 115         self.sdnc_preloading_port = '30267' if self.oom_mode else '8282'
 116         self.aai_query_port = '30233' if self.oom_mode else '8443'
 117         self.sniro_port = '30288' if self.oom_mode else '8080'
 118
 119         self.host_names = ['sdc', 'so', 'sdnc', 'robot', 'aai-inst1', self.dcae_ves_collector_name]
 120         if extra_host_names:
 121             self.host_names.extend(extra_host_names)
 122         # get IP addresses
 123         self.hosts = self.get_vm_ip(self.host_names, self.external_net_addr, self.external_net_prefix_len)
 124         # this is the keyword used to name vgw stack, must not be used in other stacks
 125         self.vgw_name_keyword = 'base_vcpe_vgw'
 126         # this is the file that will keep the index of last assigned SO name
 127         self.vgw_vfmod_name_index_file= '__var/vgw_vfmod_name_index'
 128         self.svc_instance_uuid_file = '__var/svc_instance_uuid'
 129         self.preload_dict_file = '__var/preload_dict'
 130         self.vgmux_vnf_name_file = '__var/vgmux_vnf_name'
 131         self.product_family_id = 'f9457e8c-4afd-45da-9389-46acd9bf5116'
 132         self.custom_product_family_id = 'a9a77d5a-123e-4ca2-9eb9-0b015d2ee0fb'
 133         self.instance_name_prefix = {
 134             'service': 'vcpe_svc',
 135             'network': 'vcpe_net',
 136             'vnf': 'vcpe_vnf',
 137             'vfmodule': 'vcpe_vfmodule'
 138         }
 139         self.aai_userpass = 'AAI', 'AAI'
 140
 141         ############################################################################################################
 142         # following key is overriding public key from vCPE heat templates, it's important to use correct one in here
 143         # CHANGEME part
 144         self.pub_key = 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKXDgoo3+WOqcUG8/5uUbk81+yczgwC4Y8ywTmuQqbNxlY1oQ0YxdMUqUnhitSXs5S/yRuAVOYHwGg2mCs20oAINrP+mxBI544AMIb9itPjCtgqtE2EWo6MmnFGbHB4Sx3XioE7F4VPsh7japsIwzOjbrQe+Mua1TGQ5d4nfEOQaaglXLLPFfuc7WbhbJbK6Q7rHqZfRcOwAMXgDoBqlyqKeiKwnumddo2RyNT8ljYmvB6buz7KnMinzo7qB0uktVT05FH9Rg0CTWH5norlG5qXgP2aukL0gk1ph8iAt7uYLf1ktp+LJI2gaF6L0/qli9EmVCSLr1uJ38Q8CBflhkh'
 145
 146         self.os_tenant_id = self.cloud['--os-tenant-id']
 147         self.os_region_name = self.cloud['--os-region-name']
 148         self.common_preload_config['pub_key'] = self.pub_key
 149         self.sniro_url = 'http://' + self.hosts['robot'] + ':' + self.sniro_port + '/__admin/mappings'
 150         self.sniro_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 151         self.homing_solution = 'sniro'  # value is either 'sniro' or 'oof'
 152 #        self.homing_solution = 'oof'
 153         self.customer_location_used_by_oof = {
 154             "customerLatitude": "32.897480",
 155             "customerLongitude": "-97.040443",
 156             "customerName": "some_company"
 157         }
 158
 159         #############################################################################################
 160         # SDC urls
 161         self.sdc_be_port = '30204'
 162         self.sdc_be_request_userpass = 'vid', 'Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U'
 163         self.sdc_be_request_headers = {'X-ECOMP-InstanceID': 'VID'}
 164         self.sdc_be_url_prefix = 'https://' + self.hosts['sdc'] + ':' + self.sdc_be_port
 165         self.sdc_service_list_url = self.sdc_be_url_prefix + '/sdc/v1/catalog/services'
 166
 167         self.sdc_fe_port = '30207'
 168         self.sdc_fe_request_userpass = 'beep', 'boop'
 169         self.sdc_fe_request_headers = {'USER_ID': 'demo', 'Content-Type': 'application/json'}
 170         self.sdc_fe_url_prefix = 'https://' + self.hosts['sdc'] + ':' + self.sdc_fe_port
 171         self.sdc_get_category_list_url = self.sdc_fe_url_prefix + '/sdc1/feProxy/rest/v1/categories'
 172         self.sdc_create_allotted_resource_subcategory_url = self.sdc_fe_url_prefix + '/sdc1/feProxy/rest/v1/category/resources/resourceNewCategory.allotted%20resource/subCategory'
 173
 174         #############################################################################################
 175         # SDNC urls
 176         self.sdnc_userpass = 'admin', 'Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U'
 177         self.sdnc_db_name = 'sdnctl'
 178         self.sdnc_db_user = 'sdnctl'
 179         self.sdnc_db_pass = 'gamma'
 180         self.sdnc_db_port = '32774'
 181         self.sdnc_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 182         self.sdnc_preload_network_url = 'https://' + self.hosts['sdnc'] + \
 183                                         ':' + self.sdnc_preloading_port + '/restconf/operations/VNF-API:preload-network-topology-operation'
 184         self.sdnc_preload_network_gra_url = 'https://' + self.hosts['sdnc'] + \
 185                                         ':' + self.sdnc_preloading_port + '/restconf/operations/GENERIC-RESOURCE-API:preload-network-topology-operation'
 186         self.sdnc_preload_vnf_url = 'https://' + self.hosts['sdnc'] + \
 187                                     ':' + self.sdnc_preloading_port + '/restconf/operations/VNF-API:preload-vnf-topology-operation'
 188         self.sdnc_preload_gra_url = 'https://' + self.hosts['sdnc'] + \
 189                                     ':' + self.sdnc_preloading_port + '/restconf/operations/GENERIC-RESOURCE-API:preload-vf-module-topology-operation'
 190         self.sdnc_ar_cleanup_url = 'https://' + self.hosts['sdnc'] + ':' + self.sdnc_preloading_port + \
 191                                    '/restconf/config/GENERIC-RESOURCE-API:'
 192
 193         #############################################################################################
 194         # SO urls, note: do NOT add a '/' at the end of the url
 195         self.so_req_api_url = {'v4': 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/serviceInstantiation/v7/serviceInstances',
 196                            'v5': 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/serviceInstantiation/v7/serviceInstances'}
 197         self.so_check_progress_api_url = 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/orchestrationRequests/v6'
 198         self.so_userpass = 'InfraPortalClient', 'password1$'
 199         self.so_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 200         self.so_db_name = 'catalogdb'
 201         self.so_db_user = 'root'
 202         self.so_db_pass = 'secretpassword'
 203         self.so_db_port = '30252' if self.oom_mode else '32769'
 204
 205         self.vpp_inf_url = 'http://{0}:8183/restconf/config/ietf-interfaces:interfaces'
 206         self.vpp_api_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 207         self.vpp_api_userpass = ('admin', 'admin')
 208         self.vpp_ves_url= 'http://{0}:8183/restconf/config/vesagent:vesagent'
 209
 210         #############################################################################################
 211         # POLICY urls
 212         self.policy_userpass = ('healthcheck', 'zb!XztG34')
 213         self.policy_headers = {'Accept': 'application/json', 'Content-Type': 'application/json'}
 214         self.policy_api_url = 'https://{0}:6969/policy/api/v1/policytypes/onap.policies.controlloop.Operational/versions/1.0.0/policies'
 215         self.policy_pap_get_url = 'https://{0}:6969/policy/pap/v1/pdps'
 216         self.policy_pap_json = {'policies': [{'policy-id': 'operational.vcpe'}]}
 217         self.policy_pap_post_url = self.policy_pap_get_url + '/policies'
 218         self.policy_api_service_name = 'policy-api'
 219         self.policy_pap_service_name = 'policy-pap'
 220
 221         #############################################################################################
 222         # MARIADB-GALERA settings
 223         self.mariadb_galera_endpoint_ip = self.get_k8s_service_endpoint_info('mariadb-galera','ip')
 224         self.mariadb_galera_endpoint_port = self.get_k8s_service_endpoint_info('mariadb-galera','port')
 225
 226     def heatbridge(self, openstack_stack_name, svc_instance_uuid):
 227         """
 228         Add vserver information to AAI
 229         """
 230         self.logger.info('Adding vServer information to AAI for {0}'.format(openstack_stack_name))
 231         if not self.oom_mode:
 232             cmd = '/opt/demo.sh heatbridge {0} {1} vCPE'.format(openstack_stack_name, svc_instance_uuid)
 233             ret = commands.getstatusoutput("ssh -i onap_dev root@{0} '{1}'".format(self.hosts['robot'], cmd))
 234             self.logger.debug('%s', ret)
 235         else:
 236             print('To add vGMUX vserver info to AAI, do the following:')
 237             print('- ssh to rancher')
 238             print('- sudo su -')
 239             print('- cd /root/oom/kubernetes/robot')
 240             print('- ./demo-k8s.sh onap heatbridge {0} {1} vCPE'.format(openstack_stack_name, svc_instance_uuid))
 241
 242     def get_brg_mac_from_sdnc(self):
 243         """
 244         Check table DHCP_MAP in the SDNC DB. Find the newly instantiated BRG MAC address.
 245         Note that there might be multiple BRGs, the most recently instantiated BRG always has the largest IP address.
 246         """
 247         cnx = mysql.connector.connect(user=self.sdnc_db_user, password=self.sdnc_db_pass, database=self.sdnc_db_name,
 248                                       host=self.hosts['sdnc'], port=self.sdnc_db_port)
 249         cursor = cnx.cursor()
 250         query = "SELECT * from DHCP_MAP"
 251         cursor.execute(query)
 252
 253         self.logger.debug('DHCP_MAP table in SDNC')
 254         mac_recent = None
 255         host = -1
 256         for mac, ip in cursor:
 257             self.logger.debug(mac + ':' + ip)
 258             this_host = int(ip.split('.')[-1])
 259             if host < this_host:
 260                 host = this_host
 261                 mac_recent = mac
 262
 263         cnx.close()
 264
 265         assert mac_recent
 266         return mac_recent
 267
 268     def execute_cmds_mariadb(self, cmds):
 269         self.execute_cmds_db(cmds, self.sdnc_db_user, self.sdnc_db_pass,
 270                              self.sdnc_db_name, self.mariadb_galera_endpoint_ip,
 271                              self.mariadb_galera_endpoint_port)
 272
 273     def execute_cmds_sdnc_db(self, cmds):
 274         self.execute_cmds_db(cmds, self.sdnc_db_user, self.sdnc_db_pass, self.sdnc_db_name,
 275                              self.hosts['sdnc'], self.sdnc_db_port)
 276
 277     def execute_cmds_so_db(self, cmds):
 278         self.execute_cmds_db(cmds, self.so_db_user, self.so_db_pass, self.so_db_name,
 279                              self.hosts['so'], self.so_db_port)
 280
 281     def execute_cmds_db(self, cmds, dbuser, dbpass, dbname, host, port):
 282         cnx = mysql.connector.connect(user=dbuser, password=dbpass, database=dbname, host=host, port=port)
 283         cursor = cnx.cursor()
 284         for cmd in cmds:
 285             self.logger.debug(cmd)
 286             cursor.execute(cmd)
 287             self.logger.debug('%s', cursor)
 288         cnx.commit()
 289         cursor.close()
 290         cnx.close()
 291
 292     def find_file(self, file_name_keyword, file_ext, search_dir):
 293         """
 294         :param file_name_keyword:  keyword used to look for the csar file, case insensitive matching, e.g, infra
 295         :param file_ext: e.g., csar, json
 296         :param search_dir path to search
 297         :return: path name of the file
 298         """
 299         file_name_keyword = file_name_keyword.lower()
 300         file_ext = file_ext.lower()
 301         if not file_ext.startswith('.'):
 302             file_ext = '.' + file_ext
 303
 304         filenamepath = None
 305         for file_name in os.listdir(search_dir):
 306             file_name_lower = file_name.lower()
 307             if file_name_keyword in file_name_lower and file_name_lower.endswith(file_ext):
 308                 if filenamepath:
 309                     self.logger.error('Multiple files found for *{0}*.{1} in '
 310                                       'directory {2}'.format(file_name_keyword, file_ext, search_dir))
 311                     sys.exit(1)
 312                 filenamepath = os.path.abspath(os.path.join(search_dir, file_name))
 313
 314         if filenamepath:
 315             return filenamepath
 316         else:
 317             self.logger.error("Cannot find *{0}*{1} in directory {2}".format(file_name_keyword, file_ext, search_dir))
 318             sys.exit(1)
 319
 320     @staticmethod
 321     def network_name_to_subnet_name(network_name):
 322         """
 323         :param network_name: example: vcpe_net_cpe_signal_201711281221
 324         :return: vcpe_net_cpe_signal_subnet_201711281221
 325         """
 326         fields = network_name.split('_')
 327         fields.insert(-1, 'subnet')
 328         return '_'.join(fields)
 329
 330     def set_network_name(self, network_name):
 331         param = ' '.join([k + ' ' + v for k, v in self.cloud.items()])
 332         openstackcmd = 'openstack ' + param
 333         cmd = ' '.join([openstackcmd, 'network set --name', network_name, 'ONAP-NW1'])
 334         os.popen(cmd)
 335
 336     def set_subnet_name(self, network_name):
 337         """
 338         Example: network_name =  vcpe_net_cpe_signal_201711281221
 339         set subnet name to vcpe_net_cpe_signal_subnet_201711281221
 340         :return:
 341         """
 342         param = ' '.join([k + ' ' + v for k, v in self.cloud.items()])
 343         openstackcmd = 'openstack ' + param
 344
 345         # expected results: | subnets | subnet_id |
 346         subnet_info = os.popen(openstackcmd + ' network show ' + network_name + ' |grep subnets').read().split('|')
 347         if len(subnet_info) > 2 and subnet_info[1].strip() == 'subnets':
 348             subnet_id = subnet_info[2].strip()
 349             subnet_name = self.network_name_to_subnet_name(network_name)
 350             cmd = ' '.join([openstackcmd, 'subnet set --name', subnet_name, subnet_id])
 351             os.popen(cmd)
 352             self.logger.info("Subnet name set to: " + subnet_name)
 353             return True
 354         else:
 355             self.logger.error("Can't get subnet info from network name: " + network_name)
 356             return False
 357
 358     def set_closed_loop_policy(self, policy_template_file):
 359         # Gather policy services cluster ips
 360         p_api_cluster_ip = self.get_k8s_service_cluster_ip(self.policy_api_service_name)
 361         p_pap_cluster_ip = self.get_k8s_service_cluster_ip(self.policy_pap_service_name)
 362
 363         # Read policy json from file
 364         with open(policy_template_file) as f:
 365             try:
 366                 policy_json = json.load(f)
 367             except ValueError:
 368                 self.logger.error(policy_template_file + " doesn't seem to contain valid JSON data")
 369                 sys.exit(1)
 370
 371         # Check policy already applied
 372         requests.packages.urllib3.disable_warnings()
 373         policy_exists_req = requests.get(self.policy_pap_get_url.format(
 374                             p_pap_cluster_ip), auth=self.policy_userpass,
 375                             verify=False, headers=self.policy_headers)
 376         if policy_exists_req.status_code != 200:
 377             self.logger.error('Failure in checking CL policy existence. '
 378                                'Policy-pap responded with HTTP code {0}'.format(
 379                                policy_exists_req.status_code))
 380             sys.exit(1)
 381
 382         try:
 383             policy_exists_json = policy_exists_req.json()
 384         except ValueError as e:
 385             self.logger.error('Policy-pap request failed: ' + e.message)
 386             sys.exit(1)
 387
 388         try:
 389             assert policy_exists_json['groups'][0]['pdpSubgroups'] \
 390                                [1]['policies'][0]['name'] != 'operational.vcpe'
 391         except AssertionError:
 392             self.logger.info('vCPE closed loop policy already exists, not applying')
 393             return
 394         except IndexError:
 395             pass # policy doesn't exist
 396
 397         # Create policy
 398         policy_create_req = requests.post(self.policy_api_url.format(
 399                             p_api_cluster_ip), auth=self.policy_userpass,
 400                             json=policy_json, verify=False,
 401                             headers=self.policy_headers)
 402         # Get the policy id from policy-api response
 403         if policy_create_req.status_code != 200:
 404             self.logger.error('Failed creating policy. Policy-api responded'
 405                               ' with HTTP code {0}'.format(policy_create_req.status_code))
 406             sys.exit(1)
 407
 408         try:
 409             policy_version = json.loads(policy_create_req.text)['policy-version']
 410         except (KeyError, ValueError):
 411             self.logger.error('Policy API response not understood:')
 412             self.logger.debug('\n' + str(policy_create_req.text))
 413
 414         # Inject the policy into Policy PAP
 415         self.policy_pap_json['policies'].append({'policy-version': policy_version})
 416         policy_insert_req = requests.post(self.policy_pap_post_url.format(
 417                             p_pap_cluster_ip), auth=self.policy_userpass,
 418                             json=self.policy_pap_json, verify=False,
 419                             headers=self.policy_headers)
 420         if policy_insert_req.status_code != 200:
 421             self.logger.error('Policy PAP request failed with HTTP code'
 422                               '{0}'.format(policy_insert_req.status_code))
 423             sys.exit(1)
 424         self.logger.info('Successully pushed closed loop Policy')
 425
 426     def is_node_in_aai(self, node_type, node_uuid):
 427         key = None
 428         search_node_type = None
 429         if node_type == 'service':
 430             search_node_type = 'service-instance'
 431             key = 'service-instance-id'
 432         elif node_type == 'vnf':
 433             search_node_type = 'generic-vnf'
 434             key = 'vnf-id'
 435         else:
 436             logging.error('Invalid node_type: ' + node_type)
 437             sys.exit(1)
 438
 439         url = 'https://{0}:{1}/aai/v11/search/nodes-query?search-node-type={2}&filter={3}:EQUALS:{4}'.format(
 440             self.hosts['aai-inst1'], self.aai_query_port, search_node_type, key, node_uuid)
 441
 442         headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'X-FromAppID': 'vCPE-Robot', 'X-TransactionId': 'get_aai_subscr'}
 443         requests.packages.urllib3.disable_warnings()
 444         r = requests.get(url, headers=headers, auth=self.aai_userpass, verify=False)
 445         response = r.json()
 446         self.logger.debug('aai query: ' + url)
 447         self.logger.debug('aai response:\n' + json.dumps(response, indent=4, sort_keys=True))
 448         return 'result-data' in response
 449
 450     @staticmethod
 451     def extract_ip_from_str(net_addr, net_addr_len, sz):
 452         """
 453         :param net_addr:  e.g. 10.5.12.0
 454         :param net_addr_len: e.g. 24
 455         :param sz: a string
 456         :return: the first IP address matching the network, e.g. 10.5.12.3
 457         """
 458         network = ipaddress.ip_network(unicode('{0}/{1}'.format(net_addr, net_addr_len)), strict=False)
 459         ip_list = re.findall(r'[0-9]+(?:\.[0-9]+){3}', sz)
 460         for ip in ip_list:
 461             this_net = ipaddress.ip_network(unicode('{0}/{1}'.format(ip, net_addr_len)), strict=False)
 462             if this_net == network:
 463                 return str(ip)
 464         return None
 465
 466     def get_pod_node_oam_ip(self, pod):
 467         """
 468         :Assuming kubectl is available and configured by default config (~/.kube/config)
 469         :param pod: pod name substring, e.g. 'sdnc-sdnc-0'
 470         :return pod's cluster node oam ip (10.0.0.0/16)
 471         """
 472         ret = None
 473         config.load_kube_config()
 474         api = client.CoreV1Api()
 475         kslogger = logging.getLogger('kubernetes')
 476         kslogger.setLevel(logging.INFO)
 477         res = api.list_pod_for_all_namespaces()
 478         for i in res.items:
 479             if pod in i.metadata.name:
 480                 self.logger.debug("found {0}\t{1}\t{2}".format(i.metadata.name, i.status.host_ip, i.spec.node_name))
 481                 ret = i.status.host_ip
 482                 break
 483
 484         if ret is None:
 485             ret = raw_input("Enter " + self.sdnc_controller_pod + " pod cluster node OAM IP address(10.0.0.0/16): ")
 486         return ret
 487
 488     def get_pod_node_public_ip(self, pod):
 489         """
 490         :Assuming kubectl is available and configured by default config (~/.kube/config)
 491         :param pod: pod name substring, e.g. 'sdnc-sdnc-0'
 492         :return pod's cluster node public ip (i.e. 10.12.0.0/16)
 493         """
 494         ret = None
 495         config.load_kube_config()
 496         api = client.CoreV1Api()
 497         kslogger = logging.getLogger('kubernetes')
 498         kslogger.setLevel(logging.INFO)
 499         res = api.list_pod_for_all_namespaces()
 500         for i in res.items:
 501             if pod in i.metadata.name:
 502                 ret = self.get_vm_public_ip_by_nova(i.spec.node_name)
 503                 self.logger.debug("found node {0} public ip: {1}".format(i.spec.node_name, ret))
 504                 break
 505
 506         if ret is None:
 507             ret = raw_input("Enter " + self.sdnc_controller_pod + " pod cluster node public IP address(i.e. " + self.external_net_addr + "): ")
 508         return ret
 509
 510     def get_vm_public_ip_by_nova(self, vm):
 511         """
 512         This method uses openstack nova api to retrieve vm public ip
 513         :param vm: vm name
 514         :return vm public ip
 515         """
 516         subnet = IPNetwork('{0}/{1}'.format(self.external_net_addr, self.external_net_prefix_len))
 517         nova = openstackclient.Client(2, self.cloud['--os-username'], self.cloud['--os-password'], self.cloud['--os-tenant-id'], self.cloud['--os-auth-url'])
 518         for i in nova.servers.list():
 519             if i.name == vm:
 520                 for k, v in i.networks.items():
 521                     for ip in v:
 522                         if IPAddress(ip) in subnet:
 523                             return ip
 524         return None
 525
 526     def get_vm_ip(self, keywords, net_addr=None, net_addr_len=None):
 527         """
 528         :param keywords: list of keywords to search for vm, e.g. ['bng', 'gmux', 'brg']
 529         :param net_addr: e.g. 10.12.5.0
 530         :param net_addr_len: e.g. 24
 531         :return: dictionary {keyword: ip}
 532         """
 533         if not net_addr:
 534             net_addr = self.external_net_addr
 535
 536         if not net_addr_len:
 537             net_addr_len = self.external_net_prefix_len
 538
 539         param = ' '.join([k + ' ' + v for k, v in self.cloud.items() if 'identity' not in k])
 540         openstackcmd = 'nova ' + param + ' list'
 541         self.logger.debug(openstackcmd)
 542
 543         results = os.popen(openstackcmd).read()
 544         all_vm_ip_dict = self.extract_vm_ip_as_dict(results, net_addr, net_addr_len)
 545         latest_vm_list = self.remove_old_vms(all_vm_ip_dict.keys(), self.cpe_vm_prefix)
 546         latest_vm_ip_dict = {vm: all_vm_ip_dict[vm] for vm in latest_vm_list}
 547         ip_dict = self.select_subset_vm_ip(latest_vm_ip_dict, keywords)
 548         if self.oom_mode:
 549             ip_dict.update(self.get_oom_onap_vm_ip(keywords))
 550
 551         if len(ip_dict) != len(keywords):
 552             self.logger.error('Cannot find all desired IP addresses for %s.', keywords)
 553             self.logger.error(json.dumps(ip_dict, indent=4, sort_keys=True))
 554             self.logger.error('Temporarily continue.. remember to check back vcpecommon.py line: 396')
 555 #            sys.exit(1)
 556         return ip_dict
 557
 558     def get_oom_onap_vm_ip(self, keywords):
 559         vm_ip = {}
 560         for vm in keywords:
 561             if vm in self.host_names:
 562                 vm_ip[vm] = self.oom_so_sdnc_aai_ip
 563         return vm_ip
 564
 565     def get_k8s_service_cluster_ip(self, service):
 566         """
 567         Returns cluster IP for a given service
 568         :param service: name of the service
 569         :return: cluster ip
 570         """
 571         config.load_kube_config()
 572         api = client.CoreV1Api()
 573         kslogger = logging.getLogger('kubernetes')
 574         kslogger.setLevel(logging.INFO)
 575         try:
 576             resp = api.read_namespaced_service(service, self.onap_namespace)
 577         except client.rest.ApiException as e:
 578             self.logger.error('Error while making k8s API request: ' + e.body)
 579             sys.exit(1)
 580
 581         return resp.spec.cluster_ip
 582
 583     def get_k8s_service_endpoint_info(self, service, subset):
 584         """
 585         Returns endpoint data for a given service and subset. If there
 586         is more than one endpoint returns data for the first one from
 587         the list that API returned.
 588         :param service: name of the service
 589         :param subset: subset name, one of "ip","port"
 590         :return: endpoint ip
 591         """
 592         config.load_kube_config()
 593         api = client.CoreV1Api()
 594         kslogger = logging.getLogger('kubernetes')
 595         kslogger.setLevel(logging.INFO)
 596         try:
 597             resp = api.read_namespaced_endpoints(service, self.onap_namespace)
 598         except client.rest.ApiException as e:
 599             self.logger.error('Error while making k8s API request: ' + e.body)
 600             sys.exit(1)
 601
 602         if subset == "ip":
 603             return resp.subsets[0].addresses[0].ip
 604         elif subset == "port":
 605             return resp.subsets[0].ports[0].port
 606         else:
 607             self.logger.error("Unsupported subset type")
 608
 609     def extract_vm_ip_as_dict(self, novalist_results, net_addr, net_addr_len):
 610         vm_ip_dict = {}
 611         for line in novalist_results.split('\n'):
 612             fields = line.split('|')
 613             if len(fields) == 8:
 614                 vm_name = fields[2]
 615                 ip_info = fields[-2]
 616                 ip = self.extract_ip_from_str(net_addr, net_addr_len, ip_info)
 617                 vm_ip_dict[vm_name] = ip
 618
 619         return vm_ip_dict
 620
 621     def remove_old_vms(self, vm_list, prefix):
 622         """
 623         For vms with format name_timestamp, only keep the one with the latest timestamp.
 624         E.g.,
 625             zdcpe1cpe01brgemu01_201805222148        (drop this)
 626             zdcpe1cpe01brgemu01_201805222229        (keep this)
 627             zdcpe1cpe01gw01_201805162201
 628         """
 629         new_vm_list = []
 630         same_type_vm_dict = {}
 631         for vm in vm_list:
 632             fields = vm.split('_')
 633             if vm.startswith(prefix) and len(fields) == 2 and len(fields[-1]) == len('201805222148') and fields[-1].isdigit():
 634                 if vm > same_type_vm_dict.get(fields[0], '0'):
 635                     same_type_vm_dict[fields[0]] = vm
 636             else:
 637                 new_vm_list.append(vm)
 638
 639         new_vm_list.extend(same_type_vm_dict.values())
 640         return new_vm_list
 641
 642     def select_subset_vm_ip(self, all_vm_ip_dict, vm_name_keyword_list):
 643         vm_ip_dict = {}
 644         for keyword in vm_name_keyword_list:
 645             for vm, ip in all_vm_ip_dict.items():
 646                 if keyword in vm:
 647                     vm_ip_dict[keyword] = ip
 648                     break
 649         return vm_ip_dict
 650
 651     def del_vgmux_ves_mode(self):
 652         url = self.vpp_ves_url.format(self.hosts['mux']) + '/mode'
 653         r = requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 654         self.logger.debug('%s', r)
 655
 656     def del_vgmux_ves_collector(self):
 657         url = self.vpp_ves_url.format(self.hosts['mux']) + '/config'
 658         r = requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 659         self.logger.debug('%s', r)
 660
 661     def set_vgmux_ves_collector(self ):
 662         url = self.vpp_ves_url.format(self.hosts['mux'])
 663         data = {'config':
 664                     {'server-addr': self.hosts[self.dcae_ves_collector_name],
 665                      'server-port': '30235' if self.oom_mode else '8081',
 666                      'read-interval': '10',
 667                      'is-add':'1'
 668                      }
 669                 }
 670         r = requests.post(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass, json=data)
 671         self.logger.debug('%s', r)
 672
 673     def set_vgmux_packet_loss_rate(self, lossrate, vg_vnf_instance_name):
 674         url = self.vpp_ves_url.format(self.hosts['mux'])
 675         data = {"mode":
 676                     {"working-mode": "demo",
 677                      "base-packet-loss": str(lossrate),
 678                      "source-name": vg_vnf_instance_name
 679                      }
 680                 }
 681         r = requests.post(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass, json=data)
 682         self.logger.debug('%s', r)
 683
 684         # return all the VxLAN interface names of BRG or vGMUX based on the IP address
 685     def get_vxlan_interfaces(self, ip, print_info=False):
 686         url = self.vpp_inf_url.format(ip)
 687         self.logger.debug('url is this: %s', url)
 688         r = requests.get(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 689         data = r.json()['interfaces']['interface']
 690         if print_info:
 691             for inf in data:
 692                 if 'name' in inf and 'type' in inf and inf['type'] == 'v3po:vxlan-tunnel':
 693                     print(json.dumps(inf, indent=4, sort_keys=True))
 694
 695         return [inf['name'] for inf in data if 'name' in inf and 'type' in inf and inf['type'] == 'v3po:vxlan-tunnel']
 696
 697     # delete all VxLAN interfaces of each hosts
 698     def delete_vxlan_interfaces(self, host_dic):
 699         for host, ip in host_dic.items():
 700             deleted = False
 701             self.logger.info('{0}: Getting VxLAN interfaces'.format(host))
 702             inf_list = self.get_vxlan_interfaces(ip)
 703             for inf in inf_list:
 704                 deleted = True
 705                 time.sleep(2)
 706                 self.logger.info("{0}: Deleting VxLAN crossconnect {1}".format(host, inf))
 707                 url = self.vpp_inf_url.format(ip) + '/interface/' + inf + '/v3po:l2'
 708                 requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 709
 710             for inf in inf_list:
 711                 deleted = True
 712                 time.sleep(2)
 713                 self.logger.info("{0}: Deleting VxLAN interface {1}".format(host, inf))
 714                 url = self.vpp_inf_url.format(ip) + '/interface/' + inf
 715                 requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 716
 717             if len(self.get_vxlan_interfaces(ip)) > 0:
 718                 self.logger.error("Error deleting VxLAN from {0}, try to restart the VM, IP is {1}.".format(host, ip))
 719                 return False
 720
 721             if not deleted:
 722                 self.logger.info("{0}: no VxLAN interface found, nothing to delete".format(host))
 723         return True
 724
 725     @staticmethod
 726     def save_object(obj, filepathname):
 727         with open(filepathname, 'wb') as fout:
 728             pickle.dump(obj, fout)
 729
 730     @staticmethod
 731     def load_object(filepathname):
 732         with open(filepathname, 'rb') as fin:
 733             return pickle.load(fin)
 734
 735     @staticmethod
 736     def increase_ip_address_or_vni_in_template(vnf_template_file, vnf_parameter_name_list):
 737         with open(vnf_template_file) as json_input:
 738             json_data = json.load(json_input)
 739             param_list = json_data['VNF-API:input']['VNF-API:vnf-topology-information']['VNF-API:vnf-parameters']
 740             for param in param_list:
 741                 if param['vnf-parameter-name'] in vnf_parameter_name_list:
 742                     ipaddr_or_vni = param['vnf-parameter-value'].split('.')
 743                     number = int(ipaddr_or_vni[-1])
 744                     if 254 == number:
 745                         number = 10
 746                     else:
 747                         number = number + 1
 748                     ipaddr_or_vni[-1] = str(number)
 749                     param['vnf-parameter-value'] = '.'.join(ipaddr_or_vni)
 750
 751         assert json_data is not None
 752         with open(vnf_template_file, 'w') as json_output:
 753             json.dump(json_data, json_output, indent=4, sort_keys=True)
 754
 755     def save_preload_data(self, preload_data):
 756         self.save_object(preload_data, self.preload_dict_file)
 757
 758     def load_preload_data(self):
 759         return self.load_object(self.preload_dict_file)
 760
 761     def save_vgmux_vnf_name(self, vgmux_vnf_name):
 762         self.save_object(vgmux_vnf_name, self.vgmux_vnf_name_file)
 763
 764     def load_vgmux_vnf_name(self):
 765         return self.load_object(self.vgmux_vnf_name_file)
 766