test/vcpe/vcpecommon.py

   1 import json
   2 import logging
   3 import os
   4 import pickle
   5 import re
   6 import sys
   7
   8 import ipaddress
   9 import mysql.connector
  10 import requests
  11 import commands
  12 import time
  13 from novaclient import client as openstackclient
  14 from kubernetes import client, config
  15 from netaddr import IPAddress, IPNetwork
  16
  17 class VcpeCommon:
  18     #############################################################################################
  19     #     Start: configurations that you must change for a new ONAP installation
  20     external_net_addr = '10.12.0.0'
  21     external_net_prefix_len = 16
  22     #############################################################################################
  23     # set the openstack cloud access credentials here
  24     oom_mode = True
  25
  26     cloud = {
  27         '--os-auth-url': 'http://10.12.25.2:5000',
  28         '--os-username': 'kxi',
  29         '--os-user-domain-id': 'default',
  30         '--os-project-domain-id': 'default',
  31         '--os-tenant-id': 'bc43d50ffcb84750bac0c1707a9a765b' if oom_mode else '1e097c6713e74fd7ac8e4295e605ee1e',
  32         '--os-region-name': 'RegionOne',
  33         '--os-password': 'n3JhGMGuDzD8',
  34         '--os-project-domain-name': 'Integration-SB-03' if oom_mode else 'Integration-SB-07',
  35         '--os-identity-api-version': '3'
  36     }
  37
  38     common_preload_config = {
  39         'oam_onap_net': 'oam_network_2No2' if oom_mode else 'oam_onap_lAky',
  40         'oam_onap_subnet': 'oam_network_2No2' if oom_mode else 'oam_onap_lAky',
  41         'public_net': 'external',
  42         'public_net_id': '971040b2-7059-49dc-b220-4fab50cb2ad4'
  43     }
  44     sdnc_controller_pod = 'dev-sdnc-sdnc-0'
  45
  46     #############################################################################################
  47
  48     template_variable_symbol = '${'
  49     cpe_vm_prefix = 'zdcpe'
  50     #############################################################################################
  51     # preloading network config
  52     #  key=network role
  53     #  value = [subnet_start_ip, subnet_gateway_ip]
  54     preload_network_config = {
  55         'cpe_public': ['10.2.0.2', '10.2.0.1'],
  56         'cpe_signal': ['10.4.0.2', '10.4.0.1'],
  57         'brg_bng': ['10.3.0.2', '10.3.0.1'],
  58         'bng_mux': ['10.1.0.10', '10.1.0.1'],
  59         'mux_gw': ['10.5.0.10', '10.5.0.1']
  60     }
  61
  62     dcae_ves_collector_name = 'dcae-bootstrap'
  63     global_subscriber_id = 'SDN-ETHERNET-INTERNET'
  64     project_name = 'Project-Demonstration'
  65     owning_entity_id = '520cc603-a3c4-4ec2-9ef4-ca70facd79c0'
  66     owning_entity_name = 'OE-Demonstration1'
  67
  68     def __init__(self, extra_host_names=None):
  69         self.logger = logging.getLogger(__name__)
  70         self.logger.setLevel(logging.DEBUG)
  71         self.logger.info('Initializing configuration')
  72
  73         # CHANGEME: vgw_VfModuleModelInvariantUuid is in rescust service csar, look in service-VcpesvcRescust1118-template.yml for groups vgw module metadata. TODO: read this value automcatically
  74         self.vgw_VfModuleModelInvariantUuid = '26d6a718-17b2-4ba8-8691-c44343b2ecd2'
  75         # CHANGEME: OOM: this is the address that the brg and bng will nat for sdnc access - 10.0.0.x address of k8 host for sdnc-0 container
  76         self.sdnc_oam_ip = self.get_pod_node_oam_ip('sdnc-sdnc-0')
  77         # CHANGEME: OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
  78         self.oom_so_sdnc_aai_ip = self.get_pod_node_public_ip('sdnc-sdnc-0')
  79         # CHANGEME: OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
  80         self.oom_dcae_ves_collector = self.oom_so_sdnc_aai_ip
  81         # CHANGEME: OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
  82         self.mr_ip_addr = self.oom_so_sdnc_aai_ip
  83         self.mr_ip_port = '30227'
  84         self.so_nbi_port = '30277' if self.oom_mode else '8080'
  85         self.sdnc_preloading_port = '30202' if self.oom_mode else '8282'
  86         self.aai_query_port = '30233' if self.oom_mode else '8443'
  87         self.sniro_port = '30288' if self.oom_mode else '8080'
  88
  89         self.host_names = ['sdc', 'so', 'sdnc', 'robot', 'aai-inst1', self.dcae_ves_collector_name]
  90         if extra_host_names:
  91             self.host_names.extend(extra_host_names)
  92         # get IP addresses
  93         self.hosts = self.get_vm_ip(self.host_names, self.external_net_addr, self.external_net_prefix_len)
  94         # this is the keyword used to name vgw stack, must not be used in other stacks
  95         self.vgw_name_keyword = 'base_vcpe_vgw'
  96         # this is the file that will keep the index of last assigned SO name
  97         self.vgw_vfmod_name_index_file= '__var/vgw_vfmod_name_index'
  98         self.svc_instance_uuid_file = '__var/svc_instance_uuid'
  99         self.preload_dict_file = '__var/preload_dict'
 100         self.vgmux_vnf_name_file = '__var/vgmux_vnf_name'
 101         self.product_family_id = 'f9457e8c-4afd-45da-9389-46acd9bf5116'
 102         self.custom_product_family_id = 'a9a77d5a-123e-4ca2-9eb9-0b015d2ee0fb'
 103         self.instance_name_prefix = {
 104             'service': 'vcpe_svc',
 105             'network': 'vcpe_net',
 106             'vnf': 'vcpe_vnf',
 107             'vfmodule': 'vcpe_vfmodule'
 108         }
 109         self.aai_userpass = 'AAI', 'AAI'
 110         self.pub_key = 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKXDgoo3+WOqcUG8/5uUbk81+yczgwC4Y8ywTmuQqbNxlY1oQ0YxdMUqUnhitSXs5S/yRuAVOYHwGg2mCs20oAINrP+mxBI544AMIb9itPjCtgqtE2EWo6MmnFGbHB4Sx3XioE7F4VPsh7japsIwzOjbrQe+Mua1TGQ5d4nfEOQaaglXLLPFfuc7WbhbJbK6Q7rHqZfRcOwAMXgDoBqlyqKeiKwnumddo2RyNT8ljYmvB6buz7KnMinzo7qB0uktVT05FH9Rg0CTWH5norlG5qXgP2aukL0gk1ph8iAt7uYLf1ktp+LJI2gaF6L0/qli9EmVCSLr1uJ38Q8CBflhkh'
 111         self.os_tenant_id = self.cloud['--os-tenant-id']
 112         self.os_region_name = self.cloud['--os-region-name']
 113         self.common_preload_config['pub_key'] = self.pub_key
 114         self.sniro_url = 'http://' + self.hosts['robot'] + ':' + self.sniro_port + '/__admin/mappings'
 115         self.sniro_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 116         self.homing_solution = 'sniro'  # value is either 'sniro' or 'oof'
 117 #        self.homing_solution = 'oof'
 118         self.customer_location_used_by_oof = {
 119             "customerLatitude": "32.897480",
 120             "customerLongitude": "-97.040443",
 121             "customerName": "some_company"
 122         }
 123
 124         #############################################################################################
 125         # SDC urls
 126         self.sdc_be_port = '30205'
 127         self.sdc_be_request_userpass = 'vid', 'Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U'
 128         self.sdc_be_request_headers = {'X-ECOMP-InstanceID': 'VID'}
 129         self.sdc_be_url_prefix = 'http://' + self.hosts['sdc'] + ':' + self.sdc_be_port
 130         self.sdc_service_list_url = self.sdc_be_url_prefix + '/sdc/v1/catalog/services'
 131
 132         self.sdc_fe_port = '30206'
 133         self.sdc_fe_request_userpass = 'beep', 'boop'
 134         self.sdc_fe_request_headers = {'USER_ID': 'demo', 'Content-Type': 'application/json'}
 135         self.sdc_fe_url_prefix = 'http://' + self.hosts['sdc'] + ':' + self.sdc_fe_port
 136         self.sdc_get_category_list_url = self.sdc_fe_url_prefix + '/sdc1/feProxy/rest/v1/categories'
 137         self.sdc_create_allotted_resource_subcategory_url = self.sdc_fe_url_prefix + '/sdc1/feProxy/rest/v1/category/resources/resourceNewCategory.allotted%20resource/subCategory'
 138
 139         #############################################################################################
 140         # SDNC urls
 141         self.sdnc_userpass = 'admin', 'Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U'
 142         self.sdnc_db_name = 'sdnctl'
 143         self.sdnc_db_user = 'sdnctl'
 144         self.sdnc_db_pass = 'gamma'
 145         self.sdnc_db_port = '32774'
 146         self.sdnc_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 147         self.sdnc_preload_network_url = 'http://' + self.hosts['sdnc'] + \
 148                                         ':' + self.sdnc_preloading_port + '/restconf/operations/VNF-API:preload-network-topology-operation'
 149         self.sdnc_preload_vnf_url = 'http://' + self.hosts['sdnc'] + \
 150                                     ':' + self.sdnc_preloading_port + '/restconf/operations/VNF-API:preload-vnf-topology-operation'
 151         self.sdnc_preload_gra_url = 'http://' + self.hosts['sdnc'] + \
 152                                     ':' + self.sdnc_preloading_port + '/restconf/operations/GENERIC-RESOURCE-API:preload-vf-module-topology-operation'
 153         self.sdnc_ar_cleanup_url = 'http://' + self.hosts['sdnc'] + ':' + self.sdnc_preloading_port + \
 154                                    '/restconf/config/GENERIC-RESOURCE-API:'
 155
 156         #############################################################################################
 157         # SO urls, note: do NOT add a '/' at the end of the url
 158         self.so_req_api_url = {'v4': 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/serviceInstantiation/v7/serviceInstances',
 159                            'v5': 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/serviceInstantiation/v7/serviceInstances'}
 160         self.so_check_progress_api_url = 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/orchestrationRequests/v6'
 161         self.so_userpass = 'InfraPortalClient', 'password1$'
 162         self.so_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 163         self.so_db_name = 'catalogdb'
 164         self.so_db_user = 'root'
 165         self.so_db_pass = 'secretpassword'
 166         self.so_db_port = '30252' if self.oom_mode else '32769'
 167
 168         self.vpp_inf_url = 'http://{0}:8183/restconf/config/ietf-interfaces:interfaces'
 169         self.vpp_api_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 170         self.vpp_api_userpass = ('admin', 'admin')
 171         self.vpp_ves_url= 'http://{0}:8183/restconf/config/vesagent:vesagent'
 172
 173     def headbridge(self, openstack_stack_name, svc_instance_uuid):
 174         """
 175         Add vserver information to AAI
 176         """
 177         self.logger.info('Adding vServer information to AAI for {0}'.format(openstack_stack_name))
 178         if not self.oom_mode:
 179             cmd = '/opt/demo.sh heatbridge {0} {1} vCPE'.format(openstack_stack_name, svc_instance_uuid)
 180             ret = commands.getstatusoutput("ssh -i onap_dev root@{0} '{1}'".format(self.hosts['robot'], cmd))
 181             self.logger.debug('%s', ret)
 182         else:
 183             print('To add vGMUX vserver info to AAI, do the following:')
 184             print('- ssh to rancher')
 185             print('- sudo su -')
 186             print('- cd /root/oom/kubernetes/robot')
 187             print('- ./demo-k8s.sh onap heatbridge {0} {1} vCPE'.format(openstack_stack_name, svc_instance_uuid))
 188
 189     def get_brg_mac_from_sdnc(self):
 190         """
 191         Check table DHCP_MAP in the SDNC DB. Find the newly instantiated BRG MAC address.
 192         Note that there might be multiple BRGs, the most recently instantiated BRG always has the largest IP address.
 193         """
 194         cnx = mysql.connector.connect(user=self.sdnc_db_user, password=self.sdnc_db_pass, database=self.sdnc_db_name,
 195                                       host=self.hosts['sdnc'], port=self.sdnc_db_port)
 196         cursor = cnx.cursor()
 197         query = "SELECT * from DHCP_MAP"
 198         cursor.execute(query)
 199
 200         self.logger.debug('DHCP_MAP table in SDNC')
 201         mac_recent = None
 202         host = -1
 203         for mac, ip in cursor:
 204             self.logger.debug(mac + ':' + ip)
 205             this_host = int(ip.split('.')[-1])
 206             if host < this_host:
 207                 host = this_host
 208                 mac_recent = mac
 209
 210         cnx.close()
 211
 212         assert mac_recent
 213         return mac_recent
 214
 215     def execute_cmds_sdnc_db(self, cmds):
 216         self.execute_cmds_db(cmds, self.sdnc_db_user, self.sdnc_db_pass, self.sdnc_db_name,
 217                              self.hosts['sdnc'], self.sdnc_db_port)
 218
 219     def execute_cmds_so_db(self, cmds):
 220         self.execute_cmds_db(cmds, self.so_db_user, self.so_db_pass, self.so_db_name,
 221                              self.hosts['so'], self.so_db_port)
 222
 223     def execute_cmds_db(self, cmds, dbuser, dbpass, dbname, host, port):
 224         cnx = mysql.connector.connect(user=dbuser, password=dbpass, database=dbname, host=host, port=port)
 225         cursor = cnx.cursor()
 226         for cmd in cmds:
 227             self.logger.debug(cmd)
 228             cursor.execute(cmd)
 229             self.logger.debug('%s', cursor)
 230         cnx.commit()
 231         cursor.close()
 232         cnx.close()
 233
 234     def find_file(self, file_name_keyword, file_ext, search_dir):
 235         """
 236         :param file_name_keyword:  keyword used to look for the csar file, case insensitive matching, e.g, infra
 237         :param file_ext: e.g., csar, json
 238         :param search_dir path to search
 239         :return: path name of the file
 240         """
 241         file_name_keyword = file_name_keyword.lower()
 242         file_ext = file_ext.lower()
 243         if not file_ext.startswith('.'):
 244             file_ext = '.' + file_ext
 245
 246         filenamepath = None
 247         for file_name in os.listdir(search_dir):
 248             file_name_lower = file_name.lower()
 249             if file_name_keyword in file_name_lower and file_name_lower.endswith(file_ext):
 250                 if filenamepath:
 251                     self.logger.error('Multiple files found for *{0}*.{1} in '
 252                                       'directory {2}'.format(file_name_keyword, file_ext, search_dir))
 253                     sys.exit()
 254                 filenamepath = os.path.abspath(os.path.join(search_dir, file_name))
 255
 256         if filenamepath:
 257             return filenamepath
 258         else:
 259             self.logger.error("Cannot find *{0}*{1} in directory {2}".format(file_name_keyword, file_ext, search_dir))
 260             sys.exit()
 261
 262     @staticmethod
 263     def network_name_to_subnet_name(network_name):
 264         """
 265         :param network_name: example: vcpe_net_cpe_signal_201711281221
 266         :return: vcpe_net_cpe_signal_subnet_201711281221
 267         """
 268         fields = network_name.split('_')
 269         fields.insert(-1, 'subnet')
 270         return '_'.join(fields)
 271
 272     def set_network_name(self, network_name):
 273         param = ' '.join([k + ' ' + v for k, v in self.cloud.items()])
 274         openstackcmd = 'openstack ' + param
 275         cmd = ' '.join([openstackcmd, 'network set --name', network_name, 'ONAP-NW1'])
 276         os.popen(cmd)
 277
 278     def set_subnet_name(self, network_name):
 279         """
 280         Example: network_name =  vcpe_net_cpe_signal_201711281221
 281         set subnet name to vcpe_net_cpe_signal_subnet_201711281221
 282         :return:
 283         """
 284         param = ' '.join([k + ' ' + v for k, v in self.cloud.items()])
 285         openstackcmd = 'openstack ' + param
 286
 287         # expected results: | subnets | subnet_id |
 288         subnet_info = os.popen(openstackcmd + ' network show ' + network_name + ' |grep subnets').read().split('|')
 289         if len(subnet_info) > 2 and subnet_info[1].strip() == 'subnets':
 290             subnet_id = subnet_info[2].strip()
 291             subnet_name = self.network_name_to_subnet_name(network_name)
 292             cmd = ' '.join([openstackcmd, 'subnet set --name', subnet_name, subnet_id])
 293             os.popen(cmd)
 294             self.logger.info("Subnet name set to: " + subnet_name)
 295             return True
 296         else:
 297             self.logger.error("Can't get subnet info from network name: " + network_name)
 298             return False
 299
 300     def is_node_in_aai(self, node_type, node_uuid):
 301         key = None
 302         search_node_type = None
 303         if node_type == 'service':
 304             search_node_type = 'service-instance'
 305             key = 'service-instance-id'
 306         elif node_type == 'vnf':
 307             search_node_type = 'generic-vnf'
 308             key = 'vnf-id'
 309         else:
 310             logging.error('Invalid node_type: ' + node_type)
 311             sys.exit()
 312
 313         url = 'https://{0}:{1}/aai/v11/search/nodes-query?search-node-type={2}&filter={3}:EQUALS:{4}'.format(
 314             self.hosts['aai-inst1'], self.aai_query_port, search_node_type, key, node_uuid)
 315
 316         headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'X-FromAppID': 'vCPE-Robot', 'X-TransactionId': 'get_aai_subscr'}
 317         requests.packages.urllib3.disable_warnings()
 318         r = requests.get(url, headers=headers, auth=self.aai_userpass, verify=False)
 319         response = r.json()
 320         self.logger.debug('aai query: ' + url)
 321         self.logger.debug('aai response:\n' + json.dumps(response, indent=4, sort_keys=True))
 322         return 'result-data' in response
 323
 324     @staticmethod
 325     def extract_ip_from_str(net_addr, net_addr_len, sz):
 326         """
 327         :param net_addr:  e.g. 10.5.12.0
 328         :param net_addr_len: e.g. 24
 329         :param sz: a string
 330         :return: the first IP address matching the network, e.g. 10.5.12.3
 331         """
 332         network = ipaddress.ip_network(unicode('{0}/{1}'.format(net_addr, net_addr_len)), strict=False)
 333         ip_list = re.findall(r'[0-9]+(?:\.[0-9]+){3}', sz)
 334         for ip in ip_list:
 335             this_net = ipaddress.ip_network(unicode('{0}/{1}'.format(ip, net_addr_len)), strict=False)
 336             if this_net == network:
 337                 return str(ip)
 338         return None
 339
 340     def get_pod_node_oam_ip(self, pod):
 341         """
 342         :Assuming kubectl is available and configured by default config (~/.kube/config)
 343         :param pod: pod name substring, e.g. 'sdnc-sdnc-0'
 344         :return pod's cluster node oam ip (10.0.0.0/16)
 345         """
 346         ret = None
 347         config.load_kube_config()
 348         api = client.CoreV1Api()
 349         kslogger = logging.getLogger('kubernetes')
 350         kslogger.setLevel(logging.INFO)
 351         res = api.list_pod_for_all_namespaces()
 352         for i in res.items:
 353             if pod in i.metadata.name:
 354                 self.logger.debug("found {0}\t{1}\t{2}".format(i.metadata.name, i.status.host_ip, i.spec.node_name))
 355                 ret = i.status.host_ip
 356                 break
 357
 358         if ret is None:
 359             ret = raw_input("Enter sdnc-sdnc-0 pod cluster node OAM IP address(10.0.0.0/16): ")
 360         return ret
 361
 362     def get_pod_node_public_ip(self, pod):
 363         """
 364         :Assuming kubectl is available and configured by default config (~/.kube/config)
 365         :param pod: pod name substring, e.g. 'sdnc-sdnc-0'
 366         :return pod's cluster node public ip (i.e. 10.12.0.0/16)
 367         """
 368         ret = None
 369         config.load_kube_config()
 370         api = client.CoreV1Api()
 371         kslogger = logging.getLogger('kubernetes')
 372         kslogger.setLevel(logging.INFO)
 373         res = api.list_pod_for_all_namespaces()
 374         for i in res.items:
 375             if pod in i.metadata.name:
 376                 ret = self.get_vm_public_ip_by_nova(i.spec.node_name)
 377                 self.logger.debug("found node {0} public ip: {1}".format(i.spec.node_name, ret))
 378                 break
 379
 380         if ret is None:
 381             ret = raw_input("Enter sdnc-sdnc-0 pod cluster node public IP address(i.e. 10.12.0.0/16): ")
 382         return ret
 383
 384     def get_vm_public_ip_by_nova(self, vm):
 385         """
 386         This method uses openstack nova api to retrieve vm public ip
 387         :param vm: vm name
 388         :return vm public ip
 389         """
 390         subnet = IPNetwork('{0}/{1}'.format(self.external_net_addr, self.external_net_prefix_len))
 391         nova = openstackclient.Client(2, self.cloud['--os-username'], self.cloud['--os-password'], self.cloud['--os-tenant-id'], self.cloud['--os-auth-url'])
 392         for i in nova.servers.list():
 393             if i.name == vm:
 394                 for k, v in i.networks.items():
 395                     for ip in v:
 396                         if IPAddress(ip) in subnet:
 397                             return ip
 398         return None
 399
 400     def get_vm_ip(self, keywords, net_addr=None, net_addr_len=None):
 401         """
 402         :param keywords: list of keywords to search for vm, e.g. ['bng', 'gmux', 'brg']
 403         :param net_addr: e.g. 10.12.5.0
 404         :param net_addr_len: e.g. 24
 405         :return: dictionary {keyword: ip}
 406         """
 407         if not net_addr:
 408             net_addr = self.external_net_addr
 409
 410         if not net_addr_len:
 411             net_addr_len = self.external_net_prefix_len
 412
 413         param = ' '.join([k + ' ' + v for k, v in self.cloud.items() if 'identity' not in k])
 414         openstackcmd = 'nova ' + param + ' list'
 415         self.logger.debug(openstackcmd)
 416
 417         results = os.popen(openstackcmd).read()
 418         all_vm_ip_dict = self.extract_vm_ip_as_dict(results, net_addr, net_addr_len)
 419         latest_vm_list = self.remove_old_vms(all_vm_ip_dict.keys(), self.cpe_vm_prefix)
 420         latest_vm_ip_dict = {vm: all_vm_ip_dict[vm] for vm in latest_vm_list}
 421         ip_dict = self.select_subset_vm_ip(latest_vm_ip_dict, keywords)
 422         if self.oom_mode:
 423             ip_dict.update(self.get_oom_onap_vm_ip(keywords))
 424
 425         if len(ip_dict) != len(keywords):
 426             self.logger.error('Cannot find all desired IP addresses for %s.', keywords)
 427             self.logger.error(json.dumps(ip_dict, indent=4, sort_keys=True))
 428             self.logger.error('Temporarily continue.. remember to check back vcpecommon.py line: 396')
 429 #            sys.exit()
 430         return ip_dict
 431
 432     def get_oom_onap_vm_ip(self, keywords):
 433         vm_ip = {}
 434         onap_vm_list = set(['sdc', 'so', 'sdnc', 'aai-inst1', 'robot', self.dcae_ves_collector_name])
 435         for vm in keywords:
 436             if vm in onap_vm_list:
 437                 vm_ip[vm] = self.oom_so_sdnc_aai_ip
 438         return vm_ip
 439
 440     def extract_vm_ip_as_dict(self, novalist_results, net_addr, net_addr_len):
 441         vm_ip_dict = {}
 442         for line in novalist_results.split('\n'):
 443             fields = line.split('|')
 444             if len(fields) == 8:
 445                 vm_name = fields[2]
 446                 ip_info = fields[-2]
 447                 ip = self.extract_ip_from_str(net_addr, net_addr_len, ip_info)
 448                 vm_ip_dict[vm_name] = ip
 449
 450         return vm_ip_dict
 451
 452     def remove_old_vms(self, vm_list, prefix):
 453         """
 454         For vms with format name_timestamp, only keep the one with the latest timestamp.
 455         E.g.,
 456             zdcpe1cpe01brgemu01_201805222148        (drop this)
 457             zdcpe1cpe01brgemu01_201805222229        (keep this)
 458             zdcpe1cpe01gw01_201805162201
 459         """
 460         new_vm_list = []
 461         same_type_vm_dict = {}
 462         for vm in vm_list:
 463             fields = vm.split('_')
 464             if vm.startswith(prefix) and len(fields) == 2 and len(fields[-1]) == len('201805222148') and fields[-1].isdigit():
 465                 if vm > same_type_vm_dict.get(fields[0], '0'):
 466                     same_type_vm_dict[fields[0]] = vm
 467             else:
 468                 new_vm_list.append(vm)
 469
 470         new_vm_list.extend(same_type_vm_dict.values())
 471         return new_vm_list
 472
 473     def select_subset_vm_ip(self, all_vm_ip_dict, vm_name_keyword_list):
 474         vm_ip_dict = {}
 475         for keyword in vm_name_keyword_list:
 476             for vm, ip in all_vm_ip_dict.items():
 477                 if keyword in vm:
 478                     vm_ip_dict[keyword] = ip
 479                     break
 480         return vm_ip_dict
 481
 482     def del_vgmux_ves_mode(self):
 483         url = self.vpp_ves_url.format(self.hosts['mux']) + '/mode'
 484         r = requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 485         self.logger.debug('%s', r)
 486
 487     def del_vgmux_ves_collector(self):
 488         url = self.vpp_ves_url.format(self.hosts['mux']) + '/config'
 489         r = requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 490         self.logger.debug('%s', r)
 491
 492     def set_vgmux_ves_collector(self ):
 493         url = self.vpp_ves_url.format(self.hosts['mux'])
 494         data = {'config':
 495                     {'server-addr': self.hosts[self.dcae_ves_collector_name],
 496                      'server-port': '30235' if self.oom_mode else '8081',
 497                      'read-interval': '10',
 498                      'is-add':'1'
 499                      }
 500                 }
 501         r = requests.post(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass, json=data)
 502         self.logger.debug('%s', r)
 503
 504     def set_vgmux_packet_loss_rate(self, lossrate, vg_vnf_instance_name):
 505         url = self.vpp_ves_url.format(self.hosts['mux'])
 506         data = {"mode":
 507                     {"working-mode": "demo",
 508                      "base-packet-loss": str(lossrate),
 509                      "source-name": vg_vnf_instance_name
 510                      }
 511                 }
 512         r = requests.post(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass, json=data)
 513         self.logger.debug('%s', r)
 514
 515         # return all the VxLAN interface names of BRG or vGMUX based on the IP address
 516     def get_vxlan_interfaces(self, ip, print_info=False):
 517         url = self.vpp_inf_url.format(ip)
 518         self.logger.debug('url is this: %s', url)
 519         r = requests.get(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 520         data = r.json()['interfaces']['interface']
 521         if print_info:
 522             for inf in data:
 523                 if 'name' in inf and 'type' in inf and inf['type'] == 'v3po:vxlan-tunnel':
 524                     print(json.dumps(inf, indent=4, sort_keys=True))
 525
 526         return [inf['name'] for inf in data if 'name' in inf and 'type' in inf and inf['type'] == 'v3po:vxlan-tunnel']
 527
 528     # delete all VxLAN interfaces of each hosts
 529     def delete_vxlan_interfaces(self, host_dic):
 530         for host, ip in host_dic.items():
 531             deleted = False
 532             self.logger.info('{0}: Getting VxLAN interfaces'.format(host))
 533             inf_list = self.get_vxlan_interfaces(ip)
 534             for inf in inf_list:
 535                 deleted = True
 536                 time.sleep(2)
 537                 self.logger.info("{0}: Deleting VxLAN crossconnect {1}".format(host, inf))
 538                 url = self.vpp_inf_url.format(ip) + '/interface/' + inf + '/v3po:l2'
 539                 requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 540
 541             for inf in inf_list:
 542                 deleted = True
 543                 time.sleep(2)
 544                 self.logger.info("{0}: Deleting VxLAN interface {1}".format(host, inf))
 545                 url = self.vpp_inf_url.format(ip) + '/interface/' + inf
 546                 requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 547
 548             if len(self.get_vxlan_interfaces(ip)) > 0:
 549                 self.logger.error("Error deleting VxLAN from {0}, try to restart the VM, IP is {1}.".format(host, ip))
 550                 return False
 551
 552             if not deleted:
 553                 self.logger.info("{0}: no VxLAN interface found, nothing to delete".format(host))
 554         return True
 555
 556     @staticmethod
 557     def save_object(obj, filepathname):
 558         with open(filepathname, 'wb') as fout:
 559             pickle.dump(obj, fout)
 560
 561     @staticmethod
 562     def load_object(filepathname):
 563         with open(filepathname, 'rb') as fin:
 564             return pickle.load(fin)
 565
 566     @staticmethod
 567     def increase_ip_address_or_vni_in_template(vnf_template_file, vnf_parameter_name_list):
 568         with open(vnf_template_file) as json_input:
 569             json_data = json.load(json_input)
 570             param_list = json_data['VNF-API:input']['VNF-API:vnf-topology-information']['VNF-API:vnf-parameters']
 571             for param in param_list:
 572                 if param['vnf-parameter-name'] in vnf_parameter_name_list:
 573                     ipaddr_or_vni = param['vnf-parameter-value'].split('.')
 574                     number = int(ipaddr_or_vni[-1])
 575                     if 254 == number:
 576                         number = 10
 577                     else:
 578                         number = number + 1
 579                     ipaddr_or_vni[-1] = str(number)
 580                     param['vnf-parameter-value'] = '.'.join(ipaddr_or_vni)
 581
 582         assert json_data is not None
 583         with open(vnf_template_file, 'w') as json_output:
 584             json.dump(json_data, json_output, indent=4, sort_keys=True)
 585
 586     def save_preload_data(self, preload_data):
 587         self.save_object(preload_data, self.preload_dict_file)
 588
 589     def load_preload_data(self):
 590         return self.load_object(self.preload_dict_file)
 591
 592     def save_vgmux_vnf_name(self, vgmux_vnf_name):
 593         self.save_object(vgmux_vnf_name, self.vgmux_vnf_name_file)
 594
 595     def load_vgmux_vnf_name(self):
 596         return self.load_object(self.vgmux_vnf_name_file)
 597