test/vcpe/vcpecommon.py

   1 import json
   2 import logging
   3 import os
   4 import pickle
   5 import re
   6 import sys
   7
   8 import ipaddress
   9 import mysql.connector
  10 import requests
  11 import commands
  12 import time
  13 from novaclient import client as openstackclient
  14 from kubernetes import client, config
  15 from netaddr import IPAddress, IPNetwork
  16
  17 class VcpeCommon:
  18     #############################################################################################
  19     #     Start: configurations that you must change for a new ONAP installation
  20     external_net_addr = '10.12.0.0'
  21     external_net_prefix_len = 16
  22     #############################################################################################
  23     # set the openstack cloud access credentials here
  24     oom_mode = True
  25
  26     cloud = {
  27         '--os-auth-url': 'http://10.12.25.2:5000',
  28         '--os-username': 'kxi',
  29         '--os-user-domain-id': 'default',
  30         '--os-project-domain-id': 'default',
  31         '--os-tenant-id': 'bc43d50ffcb84750bac0c1707a9a765b' if oom_mode else '1e097c6713e74fd7ac8e4295e605ee1e',
  32         '--os-region-name': 'RegionOne',
  33         '--os-password': 'n3JhGMGuDzD8',
  34         '--os-project-domain-name': 'Integration-SB-03' if oom_mode else 'Integration-SB-07',
  35         '--os-identity-api-version': '3'
  36     }
  37
  38     common_preload_config = {
  39         'oam_onap_net': 'oam_network_2No2' if oom_mode else 'oam_onap_lAky',
  40         'oam_onap_subnet': 'oam_network_2No2' if oom_mode else 'oam_onap_lAky',
  41         'public_net': 'external',
  42         'public_net_id': '971040b2-7059-49dc-b220-4fab50cb2ad4'
  43     }
  44     sdnc_controller_pod = 'dev-sdnc-sdnc-0'
  45
  46     #############################################################################################
  47
  48     template_variable_symbol = '${'
  49     cpe_vm_prefix = 'zdcpe'
  50     #############################################################################################
  51     # preloading network config
  52     #  key=network role
  53     #  value = [subnet_start_ip, subnet_gateway_ip]
  54     preload_network_config = {
  55         'cpe_public': ['10.2.0.2', '10.2.0.1'],
  56         'cpe_signal': ['10.4.0.2', '10.4.0.1'],
  57         'brg_bng': ['10.3.0.2', '10.3.0.1'],
  58         'bng_mux': ['10.1.0.10', '10.1.0.1'],
  59         'mux_gw': ['10.5.0.10', '10.5.0.1']
  60     }
  61
  62     dcae_ves_collector_name = 'dcae-bootstrap'
  63     global_subscriber_id = 'SDN-ETHERNET-INTERNET'
  64     project_name = 'Project-Demonstration'
  65     owning_entity_id = '520cc603-a3c4-4ec2-9ef4-ca70facd79c0'
  66     owning_entity_name = 'OE-Demonstration1'
  67
  68     def __init__(self, extra_host_names=None):
  69         rootlogger = logging.getLogger()
  70         handler = logging.StreamHandler()
  71         formatter = logging.Formatter('%(asctime)s %(levelname)s %(name)s.%(funcName)s(): %(message)s')
  72         handler.setFormatter(formatter)
  73         rootlogger.addHandler(handler)
  74         rootlogger.setLevel(logging.INFO)
  75
  76         self.logger = logging.getLogger(__name__)
  77         self.logger.propagate = False
  78         self.logger.addHandler(handler)
  79         self.logger.setLevel(logging.DEBUG)
  80         self.logger.info('Initializing configuration')
  81
  82         # CHANGEME: vgw_VfModuleModelInvariantUuid is in rescust service csar, look in service-VcpesvcRescust1118-template.yml for groups vgw module metadata. TODO: read this value automcatically
  83         self.vgw_VfModuleModelInvariantUuid = '26d6a718-17b2-4ba8-8691-c44343b2ecd2'
  84         # CHANGEME: OOM: this is the address that the brg and bng will nat for sdnc access - 10.0.0.x address of k8 host for sdnc-0 container
  85         self.sdnc_oam_ip = self.get_pod_node_oam_ip('sdnc-sdnc-0')
  86         # CHANGEME: OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
  87         self.oom_so_sdnc_aai_ip = self.get_pod_node_public_ip('sdnc-sdnc-0')
  88         # CHANGEME: OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
  89         self.oom_dcae_ves_collector = self.oom_so_sdnc_aai_ip
  90         # CHANGEME: OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
  91         self.mr_ip_addr = self.oom_so_sdnc_aai_ip
  92         self.mr_ip_port = '30227'
  93         self.so_nbi_port = '30277' if self.oom_mode else '8080'
  94         self.sdnc_preloading_port = '30202' if self.oom_mode else '8282'
  95         self.aai_query_port = '30233' if self.oom_mode else '8443'
  96         self.sniro_port = '30288' if self.oom_mode else '8080'
  97
  98         self.host_names = ['sdc', 'so', 'sdnc', 'robot', 'aai-inst1', self.dcae_ves_collector_name]
  99         if extra_host_names:
 100             self.host_names.extend(extra_host_names)
 101         # get IP addresses
 102         self.hosts = self.get_vm_ip(self.host_names, self.external_net_addr, self.external_net_prefix_len)
 103         # this is the keyword used to name vgw stack, must not be used in other stacks
 104         self.vgw_name_keyword = 'base_vcpe_vgw'
 105         # this is the file that will keep the index of last assigned SO name
 106         self.vgw_vfmod_name_index_file= '__var/vgw_vfmod_name_index'
 107         self.svc_instance_uuid_file = '__var/svc_instance_uuid'
 108         self.preload_dict_file = '__var/preload_dict'
 109         self.vgmux_vnf_name_file = '__var/vgmux_vnf_name'
 110         self.product_family_id = 'f9457e8c-4afd-45da-9389-46acd9bf5116'
 111         self.custom_product_family_id = 'a9a77d5a-123e-4ca2-9eb9-0b015d2ee0fb'
 112         self.instance_name_prefix = {
 113             'service': 'vcpe_svc',
 114             'network': 'vcpe_net',
 115             'vnf': 'vcpe_vnf',
 116             'vfmodule': 'vcpe_vfmodule'
 117         }
 118         self.aai_userpass = 'AAI', 'AAI'
 119         self.pub_key = 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKXDgoo3+WOqcUG8/5uUbk81+yczgwC4Y8ywTmuQqbNxlY1oQ0YxdMUqUnhitSXs5S/yRuAVOYHwGg2mCs20oAINrP+mxBI544AMIb9itPjCtgqtE2EWo6MmnFGbHB4Sx3XioE7F4VPsh7japsIwzOjbrQe+Mua1TGQ5d4nfEOQaaglXLLPFfuc7WbhbJbK6Q7rHqZfRcOwAMXgDoBqlyqKeiKwnumddo2RyNT8ljYmvB6buz7KnMinzo7qB0uktVT05FH9Rg0CTWH5norlG5qXgP2aukL0gk1ph8iAt7uYLf1ktp+LJI2gaF6L0/qli9EmVCSLr1uJ38Q8CBflhkh'
 120         self.os_tenant_id = self.cloud['--os-tenant-id']
 121         self.os_region_name = self.cloud['--os-region-name']
 122         self.common_preload_config['pub_key'] = self.pub_key
 123         self.sniro_url = 'http://' + self.hosts['robot'] + ':' + self.sniro_port + '/__admin/mappings'
 124         self.sniro_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 125         self.homing_solution = 'sniro'  # value is either 'sniro' or 'oof'
 126 #        self.homing_solution = 'oof'
 127         self.customer_location_used_by_oof = {
 128             "customerLatitude": "32.897480",
 129             "customerLongitude": "-97.040443",
 130             "customerName": "some_company"
 131         }
 132
 133         #############################################################################################
 134         # SDC urls
 135         self.sdc_be_port = '30205'
 136         self.sdc_be_request_userpass = 'vid', 'Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U'
 137         self.sdc_be_request_headers = {'X-ECOMP-InstanceID': 'VID'}
 138         self.sdc_be_url_prefix = 'http://' + self.hosts['sdc'] + ':' + self.sdc_be_port
 139         self.sdc_service_list_url = self.sdc_be_url_prefix + '/sdc/v1/catalog/services'
 140
 141         self.sdc_fe_port = '30206'
 142         self.sdc_fe_request_userpass = 'beep', 'boop'
 143         self.sdc_fe_request_headers = {'USER_ID': 'demo', 'Content-Type': 'application/json'}
 144         self.sdc_fe_url_prefix = 'http://' + self.hosts['sdc'] + ':' + self.sdc_fe_port
 145         self.sdc_get_category_list_url = self.sdc_fe_url_prefix + '/sdc1/feProxy/rest/v1/categories'
 146         self.sdc_create_allotted_resource_subcategory_url = self.sdc_fe_url_prefix + '/sdc1/feProxy/rest/v1/category/resources/resourceNewCategory.allotted%20resource/subCategory'
 147
 148         #############################################################################################
 149         # SDNC urls
 150         self.sdnc_userpass = 'admin', 'Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U'
 151         self.sdnc_db_name = 'sdnctl'
 152         self.sdnc_db_user = 'sdnctl'
 153         self.sdnc_db_pass = 'gamma'
 154         self.sdnc_db_port = '32774'
 155         self.sdnc_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 156         self.sdnc_preload_network_url = 'http://' + self.hosts['sdnc'] + \
 157                                         ':' + self.sdnc_preloading_port + '/restconf/operations/VNF-API:preload-network-topology-operation'
 158         self.sdnc_preload_vnf_url = 'http://' + self.hosts['sdnc'] + \
 159                                     ':' + self.sdnc_preloading_port + '/restconf/operations/VNF-API:preload-vnf-topology-operation'
 160         self.sdnc_preload_gra_url = 'http://' + self.hosts['sdnc'] + \
 161                                     ':' + self.sdnc_preloading_port + '/restconf/operations/GENERIC-RESOURCE-API:preload-vf-module-topology-operation'
 162         self.sdnc_ar_cleanup_url = 'http://' + self.hosts['sdnc'] + ':' + self.sdnc_preloading_port + \
 163                                    '/restconf/config/GENERIC-RESOURCE-API:'
 164
 165         #############################################################################################
 166         # SO urls, note: do NOT add a '/' at the end of the url
 167         self.so_req_api_url = {'v4': 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/serviceInstantiation/v7/serviceInstances',
 168                            'v5': 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/serviceInstantiation/v7/serviceInstances'}
 169         self.so_check_progress_api_url = 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/orchestrationRequests/v6'
 170         self.so_userpass = 'InfraPortalClient', 'password1$'
 171         self.so_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 172         self.so_db_name = 'catalogdb'
 173         self.so_db_user = 'root'
 174         self.so_db_pass = 'secretpassword'
 175         self.so_db_port = '30252' if self.oom_mode else '32769'
 176
 177         self.vpp_inf_url = 'http://{0}:8183/restconf/config/ietf-interfaces:interfaces'
 178         self.vpp_api_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 179         self.vpp_api_userpass = ('admin', 'admin')
 180         self.vpp_ves_url= 'http://{0}:8183/restconf/config/vesagent:vesagent'
 181
 182     def headbridge(self, openstack_stack_name, svc_instance_uuid):
 183         """
 184         Add vserver information to AAI
 185         """
 186         self.logger.info('Adding vServer information to AAI for {0}'.format(openstack_stack_name))
 187         if not self.oom_mode:
 188             cmd = '/opt/demo.sh heatbridge {0} {1} vCPE'.format(openstack_stack_name, svc_instance_uuid)
 189             ret = commands.getstatusoutput("ssh -i onap_dev root@{0} '{1}'".format(self.hosts['robot'], cmd))
 190             self.logger.debug('%s', ret)
 191         else:
 192             print('To add vGMUX vserver info to AAI, do the following:')
 193             print('- ssh to rancher')
 194             print('- sudo su -')
 195             print('- cd /root/oom/kubernetes/robot')
 196             print('- ./demo-k8s.sh onap heatbridge {0} {1} vCPE'.format(openstack_stack_name, svc_instance_uuid))
 197
 198     def get_brg_mac_from_sdnc(self):
 199         """
 200         Check table DHCP_MAP in the SDNC DB. Find the newly instantiated BRG MAC address.
 201         Note that there might be multiple BRGs, the most recently instantiated BRG always has the largest IP address.
 202         """
 203         cnx = mysql.connector.connect(user=self.sdnc_db_user, password=self.sdnc_db_pass, database=self.sdnc_db_name,
 204                                       host=self.hosts['sdnc'], port=self.sdnc_db_port)
 205         cursor = cnx.cursor()
 206         query = "SELECT * from DHCP_MAP"
 207         cursor.execute(query)
 208
 209         self.logger.debug('DHCP_MAP table in SDNC')
 210         mac_recent = None
 211         host = -1
 212         for mac, ip in cursor:
 213             self.logger.debug(mac + ':' + ip)
 214             this_host = int(ip.split('.')[-1])
 215             if host < this_host:
 216                 host = this_host
 217                 mac_recent = mac
 218
 219         cnx.close()
 220
 221         assert mac_recent
 222         return mac_recent
 223
 224     def execute_cmds_sdnc_db(self, cmds):
 225         self.execute_cmds_db(cmds, self.sdnc_db_user, self.sdnc_db_pass, self.sdnc_db_name,
 226                              self.hosts['sdnc'], self.sdnc_db_port)
 227
 228     def execute_cmds_so_db(self, cmds):
 229         self.execute_cmds_db(cmds, self.so_db_user, self.so_db_pass, self.so_db_name,
 230                              self.hosts['so'], self.so_db_port)
 231
 232     def execute_cmds_db(self, cmds, dbuser, dbpass, dbname, host, port):
 233         cnx = mysql.connector.connect(user=dbuser, password=dbpass, database=dbname, host=host, port=port)
 234         cursor = cnx.cursor()
 235         for cmd in cmds:
 236             self.logger.debug(cmd)
 237             cursor.execute(cmd)
 238             self.logger.debug('%s', cursor)
 239         cnx.commit()
 240         cursor.close()
 241         cnx.close()
 242
 243     def find_file(self, file_name_keyword, file_ext, search_dir):
 244         """
 245         :param file_name_keyword:  keyword used to look for the csar file, case insensitive matching, e.g, infra
 246         :param file_ext: e.g., csar, json
 247         :param search_dir path to search
 248         :return: path name of the file
 249         """
 250         file_name_keyword = file_name_keyword.lower()
 251         file_ext = file_ext.lower()
 252         if not file_ext.startswith('.'):
 253             file_ext = '.' + file_ext
 254
 255         filenamepath = None
 256         for file_name in os.listdir(search_dir):
 257             file_name_lower = file_name.lower()
 258             if file_name_keyword in file_name_lower and file_name_lower.endswith(file_ext):
 259                 if filenamepath:
 260                     self.logger.error('Multiple files found for *{0}*.{1} in '
 261                                       'directory {2}'.format(file_name_keyword, file_ext, search_dir))
 262                     sys.exit()
 263                 filenamepath = os.path.abspath(os.path.join(search_dir, file_name))
 264
 265         if filenamepath:
 266             return filenamepath
 267         else:
 268             self.logger.error("Cannot find *{0}*{1} in directory {2}".format(file_name_keyword, file_ext, search_dir))
 269             sys.exit()
 270
 271     @staticmethod
 272     def network_name_to_subnet_name(network_name):
 273         """
 274         :param network_name: example: vcpe_net_cpe_signal_201711281221
 275         :return: vcpe_net_cpe_signal_subnet_201711281221
 276         """
 277         fields = network_name.split('_')
 278         fields.insert(-1, 'subnet')
 279         return '_'.join(fields)
 280
 281     def set_network_name(self, network_name):
 282         param = ' '.join([k + ' ' + v for k, v in self.cloud.items()])
 283         openstackcmd = 'openstack ' + param
 284         cmd = ' '.join([openstackcmd, 'network set --name', network_name, 'ONAP-NW1'])
 285         os.popen(cmd)
 286
 287     def set_subnet_name(self, network_name):
 288         """
 289         Example: network_name =  vcpe_net_cpe_signal_201711281221
 290         set subnet name to vcpe_net_cpe_signal_subnet_201711281221
 291         :return:
 292         """
 293         param = ' '.join([k + ' ' + v for k, v in self.cloud.items()])
 294         openstackcmd = 'openstack ' + param
 295
 296         # expected results: | subnets | subnet_id |
 297         subnet_info = os.popen(openstackcmd + ' network show ' + network_name + ' |grep subnets').read().split('|')
 298         if len(subnet_info) > 2 and subnet_info[1].strip() == 'subnets':
 299             subnet_id = subnet_info[2].strip()
 300             subnet_name = self.network_name_to_subnet_name(network_name)
 301             cmd = ' '.join([openstackcmd, 'subnet set --name', subnet_name, subnet_id])
 302             os.popen(cmd)
 303             self.logger.info("Subnet name set to: " + subnet_name)
 304             return True
 305         else:
 306             self.logger.error("Can't get subnet info from network name: " + network_name)
 307             return False
 308
 309     def is_node_in_aai(self, node_type, node_uuid):
 310         key = None
 311         search_node_type = None
 312         if node_type == 'service':
 313             search_node_type = 'service-instance'
 314             key = 'service-instance-id'
 315         elif node_type == 'vnf':
 316             search_node_type = 'generic-vnf'
 317             key = 'vnf-id'
 318         else:
 319             logging.error('Invalid node_type: ' + node_type)
 320             sys.exit()
 321
 322         url = 'https://{0}:{1}/aai/v11/search/nodes-query?search-node-type={2}&filter={3}:EQUALS:{4}'.format(
 323             self.hosts['aai-inst1'], self.aai_query_port, search_node_type, key, node_uuid)
 324
 325         headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'X-FromAppID': 'vCPE-Robot', 'X-TransactionId': 'get_aai_subscr'}
 326         requests.packages.urllib3.disable_warnings()
 327         r = requests.get(url, headers=headers, auth=self.aai_userpass, verify=False)
 328         response = r.json()
 329         self.logger.debug('aai query: ' + url)
 330         self.logger.debug('aai response:\n' + json.dumps(response, indent=4, sort_keys=True))
 331         return 'result-data' in response
 332
 333     @staticmethod
 334     def extract_ip_from_str(net_addr, net_addr_len, sz):
 335         """
 336         :param net_addr:  e.g. 10.5.12.0
 337         :param net_addr_len: e.g. 24
 338         :param sz: a string
 339         :return: the first IP address matching the network, e.g. 10.5.12.3
 340         """
 341         network = ipaddress.ip_network(unicode('{0}/{1}'.format(net_addr, net_addr_len)), strict=False)
 342         ip_list = re.findall(r'[0-9]+(?:\.[0-9]+){3}', sz)
 343         for ip in ip_list:
 344             this_net = ipaddress.ip_network(unicode('{0}/{1}'.format(ip, net_addr_len)), strict=False)
 345             if this_net == network:
 346                 return str(ip)
 347         return None
 348
 349     def get_pod_node_oam_ip(self, pod):
 350         """
 351         :Assuming kubectl is available and configured by default config (~/.kube/config)
 352         :param pod: pod name substring, e.g. 'sdnc-sdnc-0'
 353         :return pod's cluster node oam ip (10.0.0.0/16)
 354         """
 355         ret = None
 356         config.load_kube_config()
 357         api = client.CoreV1Api()
 358         kslogger = logging.getLogger('kubernetes')
 359         kslogger.setLevel(logging.INFO)
 360         res = api.list_pod_for_all_namespaces()
 361         for i in res.items:
 362             if pod in i.metadata.name:
 363                 self.logger.debug("found {0}\t{1}\t{2}".format(i.metadata.name, i.status.host_ip, i.spec.node_name))
 364                 ret = i.status.host_ip
 365                 break
 366
 367         if ret is None:
 368             ret = raw_input("Enter sdnc-sdnc-0 pod cluster node OAM IP address(10.0.0.0/16): ")
 369         return ret
 370
 371     def get_pod_node_public_ip(self, pod):
 372         """
 373         :Assuming kubectl is available and configured by default config (~/.kube/config)
 374         :param pod: pod name substring, e.g. 'sdnc-sdnc-0'
 375         :return pod's cluster node public ip (i.e. 10.12.0.0/16)
 376         """
 377         ret = None
 378         config.load_kube_config()
 379         api = client.CoreV1Api()
 380         kslogger = logging.getLogger('kubernetes')
 381         kslogger.setLevel(logging.INFO)
 382         res = api.list_pod_for_all_namespaces()
 383         for i in res.items:
 384             if pod in i.metadata.name:
 385                 ret = self.get_vm_public_ip_by_nova(i.spec.node_name)
 386                 self.logger.debug("found node {0} public ip: {1}".format(i.spec.node_name, ret))
 387                 break
 388
 389         if ret is None:
 390             ret = raw_input("Enter sdnc-sdnc-0 pod cluster node public IP address(i.e. 10.12.0.0/16): ")
 391         return ret
 392
 393     def get_vm_public_ip_by_nova(self, vm):
 394         """
 395         This method uses openstack nova api to retrieve vm public ip
 396         :param vm: vm name
 397         :return vm public ip
 398         """
 399         subnet = IPNetwork('{0}/{1}'.format(self.external_net_addr, self.external_net_prefix_len))
 400         nova = openstackclient.Client(2, self.cloud['--os-username'], self.cloud['--os-password'], self.cloud['--os-tenant-id'], self.cloud['--os-auth-url'])
 401         for i in nova.servers.list():
 402             if i.name == vm:
 403                 for k, v in i.networks.items():
 404                     for ip in v:
 405                         if IPAddress(ip) in subnet:
 406                             return ip
 407         return None
 408
 409     def get_vm_ip(self, keywords, net_addr=None, net_addr_len=None):
 410         """
 411         :param keywords: list of keywords to search for vm, e.g. ['bng', 'gmux', 'brg']
 412         :param net_addr: e.g. 10.12.5.0
 413         :param net_addr_len: e.g. 24
 414         :return: dictionary {keyword: ip}
 415         """
 416         if not net_addr:
 417             net_addr = self.external_net_addr
 418
 419         if not net_addr_len:
 420             net_addr_len = self.external_net_prefix_len
 421
 422         param = ' '.join([k + ' ' + v for k, v in self.cloud.items() if 'identity' not in k])
 423         openstackcmd = 'nova ' + param + ' list'
 424         self.logger.debug(openstackcmd)
 425
 426         results = os.popen(openstackcmd).read()
 427         all_vm_ip_dict = self.extract_vm_ip_as_dict(results, net_addr, net_addr_len)
 428         latest_vm_list = self.remove_old_vms(all_vm_ip_dict.keys(), self.cpe_vm_prefix)
 429         latest_vm_ip_dict = {vm: all_vm_ip_dict[vm] for vm in latest_vm_list}
 430         ip_dict = self.select_subset_vm_ip(latest_vm_ip_dict, keywords)
 431         if self.oom_mode:
 432             ip_dict.update(self.get_oom_onap_vm_ip(keywords))
 433
 434         if len(ip_dict) != len(keywords):
 435             self.logger.error('Cannot find all desired IP addresses for %s.', keywords)
 436             self.logger.error(json.dumps(ip_dict, indent=4, sort_keys=True))
 437             self.logger.error('Temporarily continue.. remember to check back vcpecommon.py line: 396')
 438 #            sys.exit()
 439         return ip_dict
 440
 441     def get_oom_onap_vm_ip(self, keywords):
 442         vm_ip = {}
 443         onap_vm_list = set(['sdc', 'so', 'sdnc', 'aai-inst1', 'robot', self.dcae_ves_collector_name])
 444         for vm in keywords:
 445             if vm in onap_vm_list:
 446                 vm_ip[vm] = self.oom_so_sdnc_aai_ip
 447         return vm_ip
 448
 449     def extract_vm_ip_as_dict(self, novalist_results, net_addr, net_addr_len):
 450         vm_ip_dict = {}
 451         for line in novalist_results.split('\n'):
 452             fields = line.split('|')
 453             if len(fields) == 8:
 454                 vm_name = fields[2]
 455                 ip_info = fields[-2]
 456                 ip = self.extract_ip_from_str(net_addr, net_addr_len, ip_info)
 457                 vm_ip_dict[vm_name] = ip
 458
 459         return vm_ip_dict
 460
 461     def remove_old_vms(self, vm_list, prefix):
 462         """
 463         For vms with format name_timestamp, only keep the one with the latest timestamp.
 464         E.g.,
 465             zdcpe1cpe01brgemu01_201805222148        (drop this)
 466             zdcpe1cpe01brgemu01_201805222229        (keep this)
 467             zdcpe1cpe01gw01_201805162201
 468         """
 469         new_vm_list = []
 470         same_type_vm_dict = {}
 471         for vm in vm_list:
 472             fields = vm.split('_')
 473             if vm.startswith(prefix) and len(fields) == 2 and len(fields[-1]) == len('201805222148') and fields[-1].isdigit():
 474                 if vm > same_type_vm_dict.get(fields[0], '0'):
 475                     same_type_vm_dict[fields[0]] = vm
 476             else:
 477                 new_vm_list.append(vm)
 478
 479         new_vm_list.extend(same_type_vm_dict.values())
 480         return new_vm_list
 481
 482     def select_subset_vm_ip(self, all_vm_ip_dict, vm_name_keyword_list):
 483         vm_ip_dict = {}
 484         for keyword in vm_name_keyword_list:
 485             for vm, ip in all_vm_ip_dict.items():
 486                 if keyword in vm:
 487                     vm_ip_dict[keyword] = ip
 488                     break
 489         return vm_ip_dict
 490
 491     def del_vgmux_ves_mode(self):
 492         url = self.vpp_ves_url.format(self.hosts['mux']) + '/mode'
 493         r = requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 494         self.logger.debug('%s', r)
 495
 496     def del_vgmux_ves_collector(self):
 497         url = self.vpp_ves_url.format(self.hosts['mux']) + '/config'
 498         r = requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 499         self.logger.debug('%s', r)
 500
 501     def set_vgmux_ves_collector(self ):
 502         url = self.vpp_ves_url.format(self.hosts['mux'])
 503         data = {'config':
 504                     {'server-addr': self.hosts[self.dcae_ves_collector_name],
 505                      'server-port': '30235' if self.oom_mode else '8081',
 506                      'read-interval': '10',
 507                      'is-add':'1'
 508                      }
 509                 }
 510         r = requests.post(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass, json=data)
 511         self.logger.debug('%s', r)
 512
 513     def set_vgmux_packet_loss_rate(self, lossrate, vg_vnf_instance_name):
 514         url = self.vpp_ves_url.format(self.hosts['mux'])
 515         data = {"mode":
 516                     {"working-mode": "demo",
 517                      "base-packet-loss": str(lossrate),
 518                      "source-name": vg_vnf_instance_name
 519                      }
 520                 }
 521         r = requests.post(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass, json=data)
 522         self.logger.debug('%s', r)
 523
 524         # return all the VxLAN interface names of BRG or vGMUX based on the IP address
 525     def get_vxlan_interfaces(self, ip, print_info=False):
 526         url = self.vpp_inf_url.format(ip)
 527         self.logger.debug('url is this: %s', url)
 528         r = requests.get(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 529         data = r.json()['interfaces']['interface']
 530         if print_info:
 531             for inf in data:
 532                 if 'name' in inf and 'type' in inf and inf['type'] == 'v3po:vxlan-tunnel':
 533                     print(json.dumps(inf, indent=4, sort_keys=True))
 534
 535         return [inf['name'] for inf in data if 'name' in inf and 'type' in inf and inf['type'] == 'v3po:vxlan-tunnel']
 536
 537     # delete all VxLAN interfaces of each hosts
 538     def delete_vxlan_interfaces(self, host_dic):
 539         for host, ip in host_dic.items():
 540             deleted = False
 541             self.logger.info('{0}: Getting VxLAN interfaces'.format(host))
 542             inf_list = self.get_vxlan_interfaces(ip)
 543             for inf in inf_list:
 544                 deleted = True
 545                 time.sleep(2)
 546                 self.logger.info("{0}: Deleting VxLAN crossconnect {1}".format(host, inf))
 547                 url = self.vpp_inf_url.format(ip) + '/interface/' + inf + '/v3po:l2'
 548                 requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 549
 550             for inf in inf_list:
 551                 deleted = True
 552                 time.sleep(2)
 553                 self.logger.info("{0}: Deleting VxLAN interface {1}".format(host, inf))
 554                 url = self.vpp_inf_url.format(ip) + '/interface/' + inf
 555                 requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 556
 557             if len(self.get_vxlan_interfaces(ip)) > 0:
 558                 self.logger.error("Error deleting VxLAN from {0}, try to restart the VM, IP is {1}.".format(host, ip))
 559                 return False
 560
 561             if not deleted:
 562                 self.logger.info("{0}: no VxLAN interface found, nothing to delete".format(host))
 563         return True
 564
 565     @staticmethod
 566     def save_object(obj, filepathname):
 567         with open(filepathname, 'wb') as fout:
 568             pickle.dump(obj, fout)
 569
 570     @staticmethod
 571     def load_object(filepathname):
 572         with open(filepathname, 'rb') as fin:
 573             return pickle.load(fin)
 574
 575     @staticmethod
 576     def increase_ip_address_or_vni_in_template(vnf_template_file, vnf_parameter_name_list):
 577         with open(vnf_template_file) as json_input:
 578             json_data = json.load(json_input)
 579             param_list = json_data['VNF-API:input']['VNF-API:vnf-topology-information']['VNF-API:vnf-parameters']
 580             for param in param_list:
 581                 if param['vnf-parameter-name'] in vnf_parameter_name_list:
 582                     ipaddr_or_vni = param['vnf-parameter-value'].split('.')
 583                     number = int(ipaddr_or_vni[-1])
 584                     if 254 == number:
 585                         number = 10
 586                     else:
 587                         number = number + 1
 588                     ipaddr_or_vni[-1] = str(number)
 589                     param['vnf-parameter-value'] = '.'.join(ipaddr_or_vni)
 590
 591         assert json_data is not None
 592         with open(vnf_template_file, 'w') as json_output:
 593             json.dump(json_data, json_output, indent=4, sort_keys=True)
 594
 595     def save_preload_data(self, preload_data):
 596         self.save_object(preload_data, self.preload_dict_file)
 597
 598     def load_preload_data(self):
 599         return self.load_object(self.preload_dict_file)
 600
 601     def save_vgmux_vnf_name(self, vgmux_vnf_name):
 602         self.save_object(vgmux_vnf_name, self.vgmux_vnf_name_file)
 603
 604     def load_vgmux_vnf_name(self):
 605         return self.load_object(self.vgmux_vnf_name_file)
 606