test/vcpe/vcpecommon.py

   1 import json
   2 import logging
   3 import os
   4 import pickle
   5 import re
   6 import sys
   7
   8 import ipaddress
   9 import mysql.connector
  10 import requests
  11 import commands
  12 import time
  13 from novaclient import client as openstackclient
  14 from kubernetes import client, config
  15 from netaddr import IPAddress, IPNetwork
  16
  17 ######################################################################
  18 # Parts which must be updated / cross-checked during each deployment #
  19 # are marked as CHANGEME                                             #
  20 ######################################################################
  21
  22 class VcpeCommon:
  23     #############################################################################################
  24     # Set network prefix of k8s host external address; it's used for pod public IP autodetection
  25     # but can be overriden from user in case of autodetection failure
  26     external_net_addr = '10.12.0.0'
  27     external_net_prefix_len = 16
  28
  29     #############################################################################################
  30     # set the openstack cloud access credentials here
  31     oom_mode = True
  32
  33     ###########################
  34     # set Openstack credentials
  35     # CHANGEME part
  36     cloud = {
  37         '--os-auth-url': 'http://10.12.25.2:5000',
  38         '--os-username': 'kxi',
  39         '--os-user-domain-id': 'default',
  40         '--os-project-domain-id': 'default',
  41         '--os-tenant-id': 'bc43d50ffcb84750bac0c1707a9a765b' if oom_mode else '1e097c6713e74fd7ac8e4295e605ee1e',
  42         '--os-region-name': 'RegionOne',
  43         '--os-password': 'n3JhGMGuDzD8',
  44         '--os-project-domain-name': 'Integration-SB-03' if oom_mode else 'Integration-SB-07',
  45         '--os-identity-api-version': '3'
  46     }
  47
  48     ############################################################################
  49     # set oam and public network which must exist in openstack before deployment
  50     # CHANGEME part
  51     common_preload_config = {
  52         'oam_onap_net': 'oam_network_2No2' if oom_mode else 'oam_onap_lAky',
  53         'oam_onap_subnet': 'oam_network_2No2' if oom_mode else 'oam_onap_lAky',
  54         'public_net': 'external',
  55         'public_net_id': '971040b2-7059-49dc-b220-4fab50cb2ad4'
  56     }
  57
  58     #############################################################################
  59     # set name of sdnc controller pod, prefix is taken from helm environment name
  60     # CHANGEME part
  61     sdnc_controller_pod = 'dev-sdnc-sdnc-0'
  62
  63     template_variable_symbol = '${'
  64     cpe_vm_prefix = 'zdcpe'
  65
  66     #############################################################################################
  67     # preloading network config
  68     #  key=network role
  69     #  value = [subnet_start_ip, subnet_gateway_ip]
  70     preload_network_config = {
  71         'cpe_public': ['10.2.0.2', '10.2.0.1'],
  72         'cpe_signal': ['10.4.0.2', '10.4.0.1'],
  73         'brg_bng': ['10.3.0.2', '10.3.0.1'],
  74         'bng_mux': ['10.1.0.10', '10.1.0.1'],
  75         'mux_gw': ['10.5.0.10', '10.5.0.1']
  76     }
  77
  78     dcae_ves_collector_name = 'dcae-bootstrap'
  79     global_subscriber_id = 'SDN-ETHERNET-INTERNET'
  80     project_name = 'Project-Demonstration'
  81     owning_entity_id = '520cc603-a3c4-4ec2-9ef4-ca70facd79c0'
  82     owning_entity_name = 'OE-Demonstration1'
  83
  84     def __init__(self, extra_host_names=None):
  85         self.logger = logging.getLogger(__name__)
  86         self.logger.setLevel(logging.DEBUG)
  87         self.logger.info('Initializing configuration')
  88
  89         ##################################################################################################################################
  90         # following param must be updated e.g. from csar file (grep for VfModuleModelInvariantUuid string) before vcpe.py customer call !!
  91         # vgw_VfModuleModelInvariantUuid is in rescust service csar,
  92         # look in service-VcpesvcRescust1118-template.yml for groups vgw module metadata. TODO: read this value automatically
  93         # CHANGEME part
  94         self.vgw_VfModuleModelInvariantUuid = '26d6a718-17b2-4ba8-8691-c44343b2ecd2'
  95
  96         # OOM: this is the address that the brg and bng will nat for sdnc access - 10.0.0.x address of k8 host for sdnc-0 container
  97         self.sdnc_oam_ip = self.get_pod_node_oam_ip(self.sdnc_controller_pod)
  98         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
  99         self.oom_so_sdnc_aai_ip = self.get_pod_node_public_ip(self.sdnc_controller_pod)
 100         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
 101         self.oom_dcae_ves_collector = self.oom_so_sdnc_aai_ip
 102         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
 103         self.mr_ip_addr = self.oom_so_sdnc_aai_ip
 104         self.mr_ip_port = '30227'
 105         self.so_nbi_port = '30277' if self.oom_mode else '8080'
 106         self.sdnc_preloading_port = '30202' if self.oom_mode else '8282'
 107         self.aai_query_port = '30233' if self.oom_mode else '8443'
 108         self.sniro_port = '30288' if self.oom_mode else '8080'
 109
 110         self.host_names = ['sdc', 'so', 'sdnc', 'robot', 'aai-inst1', self.dcae_ves_collector_name]
 111         if extra_host_names:
 112             self.host_names.extend(extra_host_names)
 113         # get IP addresses
 114         self.hosts = self.get_vm_ip(self.host_names, self.external_net_addr, self.external_net_prefix_len)
 115         # this is the keyword used to name vgw stack, must not be used in other stacks
 116         self.vgw_name_keyword = 'base_vcpe_vgw'
 117         # this is the file that will keep the index of last assigned SO name
 118         self.vgw_vfmod_name_index_file= '__var/vgw_vfmod_name_index'
 119         self.svc_instance_uuid_file = '__var/svc_instance_uuid'
 120         self.preload_dict_file = '__var/preload_dict'
 121         self.vgmux_vnf_name_file = '__var/vgmux_vnf_name'
 122         self.product_family_id = 'f9457e8c-4afd-45da-9389-46acd9bf5116'
 123         self.custom_product_family_id = 'a9a77d5a-123e-4ca2-9eb9-0b015d2ee0fb'
 124         self.instance_name_prefix = {
 125             'service': 'vcpe_svc',
 126             'network': 'vcpe_net',
 127             'vnf': 'vcpe_vnf',
 128             'vfmodule': 'vcpe_vfmodule'
 129         }
 130         self.aai_userpass = 'AAI', 'AAI'
 131
 132         ############################################################################################################
 133         # following key is overriding public key from vCPE heat templates, it's important to use correct one in here
 134         # CHANGEME part
 135         self.pub_key = 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKXDgoo3+WOqcUG8/5uUbk81+yczgwC4Y8ywTmuQqbNxlY1oQ0YxdMUqUnhitSXs5S/yRuAVOYHwGg2mCs20oAINrP+mxBI544AMIb9itPjCtgqtE2EWo6MmnFGbHB4Sx3XioE7F4VPsh7japsIwzOjbrQe+Mua1TGQ5d4nfEOQaaglXLLPFfuc7WbhbJbK6Q7rHqZfRcOwAMXgDoBqlyqKeiKwnumddo2RyNT8ljYmvB6buz7KnMinzo7qB0uktVT05FH9Rg0CTWH5norlG5qXgP2aukL0gk1ph8iAt7uYLf1ktp+LJI2gaF6L0/qli9EmVCSLr1uJ38Q8CBflhkh'
 136
 137         self.os_tenant_id = self.cloud['--os-tenant-id']
 138         self.os_region_name = self.cloud['--os-region-name']
 139         self.common_preload_config['pub_key'] = self.pub_key
 140         self.sniro_url = 'http://' + self.hosts['robot'] + ':' + self.sniro_port + '/__admin/mappings'
 141         self.sniro_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 142         self.homing_solution = 'sniro'  # value is either 'sniro' or 'oof'
 143 #        self.homing_solution = 'oof'
 144         self.customer_location_used_by_oof = {
 145             "customerLatitude": "32.897480",
 146             "customerLongitude": "-97.040443",
 147             "customerName": "some_company"
 148         }
 149
 150         #############################################################################################
 151         # SDC urls
 152         self.sdc_be_port = '30204'
 153         self.sdc_be_request_userpass = 'vid', 'Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U'
 154         self.sdc_be_request_headers = {'X-ECOMP-InstanceID': 'VID'}
 155         self.sdc_be_url_prefix = 'https://' + self.hosts['sdc'] + ':' + self.sdc_be_port
 156         self.sdc_service_list_url = self.sdc_be_url_prefix + '/sdc/v1/catalog/services'
 157
 158         self.sdc_fe_port = '30207'
 159         self.sdc_fe_request_userpass = 'beep', 'boop'
 160         self.sdc_fe_request_headers = {'USER_ID': 'demo', 'Content-Type': 'application/json'}
 161         self.sdc_fe_url_prefix = 'https://' + self.hosts['sdc'] + ':' + self.sdc_fe_port
 162         self.sdc_get_category_list_url = self.sdc_fe_url_prefix + '/sdc1/feProxy/rest/v1/categories'
 163         self.sdc_create_allotted_resource_subcategory_url = self.sdc_fe_url_prefix + '/sdc1/feProxy/rest/v1/category/resources/resourceNewCategory.allotted%20resource/subCategory'
 164
 165         #############################################################################################
 166         # SDNC urls
 167         self.sdnc_userpass = 'admin', 'Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U'
 168         self.sdnc_db_name = 'sdnctl'
 169         self.sdnc_db_user = 'sdnctl'
 170         self.sdnc_db_pass = 'gamma'
 171         self.sdnc_db_port = '32774'
 172         self.sdnc_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 173         self.sdnc_preload_network_url = 'http://' + self.hosts['sdnc'] + \
 174                                         ':' + self.sdnc_preloading_port + '/restconf/operations/VNF-API:preload-network-topology-operation'
 175         self.sdnc_preload_vnf_url = 'http://' + self.hosts['sdnc'] + \
 176                                     ':' + self.sdnc_preloading_port + '/restconf/operations/VNF-API:preload-vnf-topology-operation'
 177         self.sdnc_preload_gra_url = 'http://' + self.hosts['sdnc'] + \
 178                                     ':' + self.sdnc_preloading_port + '/restconf/operations/GENERIC-RESOURCE-API:preload-vf-module-topology-operation'
 179         self.sdnc_ar_cleanup_url = 'http://' + self.hosts['sdnc'] + ':' + self.sdnc_preloading_port + \
 180                                    '/restconf/config/GENERIC-RESOURCE-API:'
 181
 182         #############################################################################################
 183         # SO urls, note: do NOT add a '/' at the end of the url
 184         self.so_req_api_url = {'v4': 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/serviceInstantiation/v7/serviceInstances',
 185                            'v5': 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/serviceInstantiation/v7/serviceInstances'}
 186         self.so_check_progress_api_url = 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/orchestrationRequests/v6'
 187         self.so_userpass = 'InfraPortalClient', 'password1$'
 188         self.so_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 189         self.so_db_name = 'catalogdb'
 190         self.so_db_user = 'root'
 191         self.so_db_pass = 'secretpassword'
 192         self.so_db_port = '30252' if self.oom_mode else '32769'
 193
 194         self.vpp_inf_url = 'http://{0}:8183/restconf/config/ietf-interfaces:interfaces'
 195         self.vpp_api_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 196         self.vpp_api_userpass = ('admin', 'admin')
 197         self.vpp_ves_url= 'http://{0}:8183/restconf/config/vesagent:vesagent'
 198
 199     def heatbridge(self, openstack_stack_name, svc_instance_uuid):
 200         """
 201         Add vserver information to AAI
 202         """
 203         self.logger.info('Adding vServer information to AAI for {0}'.format(openstack_stack_name))
 204         if not self.oom_mode:
 205             cmd = '/opt/demo.sh heatbridge {0} {1} vCPE'.format(openstack_stack_name, svc_instance_uuid)
 206             ret = commands.getstatusoutput("ssh -i onap_dev root@{0} '{1}'".format(self.hosts['robot'], cmd))
 207             self.logger.debug('%s', ret)
 208         else:
 209             print('To add vGMUX vserver info to AAI, do the following:')
 210             print('- ssh to rancher')
 211             print('- sudo su -')
 212             print('- cd /root/oom/kubernetes/robot')
 213             print('- ./demo-k8s.sh onap heatbridge {0} {1} vCPE'.format(openstack_stack_name, svc_instance_uuid))
 214
 215     def get_brg_mac_from_sdnc(self):
 216         """
 217         Check table DHCP_MAP in the SDNC DB. Find the newly instantiated BRG MAC address.
 218         Note that there might be multiple BRGs, the most recently instantiated BRG always has the largest IP address.
 219         """
 220         cnx = mysql.connector.connect(user=self.sdnc_db_user, password=self.sdnc_db_pass, database=self.sdnc_db_name,
 221                                       host=self.hosts['sdnc'], port=self.sdnc_db_port)
 222         cursor = cnx.cursor()
 223         query = "SELECT * from DHCP_MAP"
 224         cursor.execute(query)
 225
 226         self.logger.debug('DHCP_MAP table in SDNC')
 227         mac_recent = None
 228         host = -1
 229         for mac, ip in cursor:
 230             self.logger.debug(mac + ':' + ip)
 231             this_host = int(ip.split('.')[-1])
 232             if host < this_host:
 233                 host = this_host
 234                 mac_recent = mac
 235
 236         cnx.close()
 237
 238         assert mac_recent
 239         return mac_recent
 240
 241     def execute_cmds_sdnc_db(self, cmds):
 242         self.execute_cmds_db(cmds, self.sdnc_db_user, self.sdnc_db_pass, self.sdnc_db_name,
 243                              self.hosts['sdnc'], self.sdnc_db_port)
 244
 245     def execute_cmds_so_db(self, cmds):
 246         self.execute_cmds_db(cmds, self.so_db_user, self.so_db_pass, self.so_db_name,
 247                              self.hosts['so'], self.so_db_port)
 248
 249     def execute_cmds_db(self, cmds, dbuser, dbpass, dbname, host, port):
 250         cnx = mysql.connector.connect(user=dbuser, password=dbpass, database=dbname, host=host, port=port)
 251         cursor = cnx.cursor()
 252         for cmd in cmds:
 253             self.logger.debug(cmd)
 254             cursor.execute(cmd)
 255             self.logger.debug('%s', cursor)
 256         cnx.commit()
 257         cursor.close()
 258         cnx.close()
 259
 260     def find_file(self, file_name_keyword, file_ext, search_dir):
 261         """
 262         :param file_name_keyword:  keyword used to look for the csar file, case insensitive matching, e.g, infra
 263         :param file_ext: e.g., csar, json
 264         :param search_dir path to search
 265         :return: path name of the file
 266         """
 267         file_name_keyword = file_name_keyword.lower()
 268         file_ext = file_ext.lower()
 269         if not file_ext.startswith('.'):
 270             file_ext = '.' + file_ext
 271
 272         filenamepath = None
 273         for file_name in os.listdir(search_dir):
 274             file_name_lower = file_name.lower()
 275             if file_name_keyword in file_name_lower and file_name_lower.endswith(file_ext):
 276                 if filenamepath:
 277                     self.logger.error('Multiple files found for *{0}*.{1} in '
 278                                       'directory {2}'.format(file_name_keyword, file_ext, search_dir))
 279                     sys.exit()
 280                 filenamepath = os.path.abspath(os.path.join(search_dir, file_name))
 281
 282         if filenamepath:
 283             return filenamepath
 284         else:
 285             self.logger.error("Cannot find *{0}*{1} in directory {2}".format(file_name_keyword, file_ext, search_dir))
 286             sys.exit()
 287
 288     @staticmethod
 289     def network_name_to_subnet_name(network_name):
 290         """
 291         :param network_name: example: vcpe_net_cpe_signal_201711281221
 292         :return: vcpe_net_cpe_signal_subnet_201711281221
 293         """
 294         fields = network_name.split('_')
 295         fields.insert(-1, 'subnet')
 296         return '_'.join(fields)
 297
 298     def set_network_name(self, network_name):
 299         param = ' '.join([k + ' ' + v for k, v in self.cloud.items()])
 300         openstackcmd = 'openstack ' + param
 301         cmd = ' '.join([openstackcmd, 'network set --name', network_name, 'ONAP-NW1'])
 302         os.popen(cmd)
 303
 304     def set_subnet_name(self, network_name):
 305         """
 306         Example: network_name =  vcpe_net_cpe_signal_201711281221
 307         set subnet name to vcpe_net_cpe_signal_subnet_201711281221
 308         :return:
 309         """
 310         param = ' '.join([k + ' ' + v for k, v in self.cloud.items()])
 311         openstackcmd = 'openstack ' + param
 312
 313         # expected results: | subnets | subnet_id |
 314         subnet_info = os.popen(openstackcmd + ' network show ' + network_name + ' |grep subnets').read().split('|')
 315         if len(subnet_info) > 2 and subnet_info[1].strip() == 'subnets':
 316             subnet_id = subnet_info[2].strip()
 317             subnet_name = self.network_name_to_subnet_name(network_name)
 318             cmd = ' '.join([openstackcmd, 'subnet set --name', subnet_name, subnet_id])
 319             os.popen(cmd)
 320             self.logger.info("Subnet name set to: " + subnet_name)
 321             return True
 322         else:
 323             self.logger.error("Can't get subnet info from network name: " + network_name)
 324             return False
 325
 326     def is_node_in_aai(self, node_type, node_uuid):
 327         key = None
 328         search_node_type = None
 329         if node_type == 'service':
 330             search_node_type = 'service-instance'
 331             key = 'service-instance-id'
 332         elif node_type == 'vnf':
 333             search_node_type = 'generic-vnf'
 334             key = 'vnf-id'
 335         else:
 336             logging.error('Invalid node_type: ' + node_type)
 337             sys.exit()
 338
 339         url = 'https://{0}:{1}/aai/v11/search/nodes-query?search-node-type={2}&filter={3}:EQUALS:{4}'.format(
 340             self.hosts['aai-inst1'], self.aai_query_port, search_node_type, key, node_uuid)
 341
 342         headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'X-FromAppID': 'vCPE-Robot', 'X-TransactionId': 'get_aai_subscr'}
 343         requests.packages.urllib3.disable_warnings()
 344         r = requests.get(url, headers=headers, auth=self.aai_userpass, verify=False)
 345         response = r.json()
 346         self.logger.debug('aai query: ' + url)
 347         self.logger.debug('aai response:\n' + json.dumps(response, indent=4, sort_keys=True))
 348         return 'result-data' in response
 349
 350     @staticmethod
 351     def extract_ip_from_str(net_addr, net_addr_len, sz):
 352         """
 353         :param net_addr:  e.g. 10.5.12.0
 354         :param net_addr_len: e.g. 24
 355         :param sz: a string
 356         :return: the first IP address matching the network, e.g. 10.5.12.3
 357         """
 358         network = ipaddress.ip_network(unicode('{0}/{1}'.format(net_addr, net_addr_len)), strict=False)
 359         ip_list = re.findall(r'[0-9]+(?:\.[0-9]+){3}', sz)
 360         for ip in ip_list:
 361             this_net = ipaddress.ip_network(unicode('{0}/{1}'.format(ip, net_addr_len)), strict=False)
 362             if this_net == network:
 363                 return str(ip)
 364         return None
 365
 366     def get_pod_node_oam_ip(self, pod):
 367         """
 368         :Assuming kubectl is available and configured by default config (~/.kube/config)
 369         :param pod: pod name substring, e.g. 'sdnc-sdnc-0'
 370         :return pod's cluster node oam ip (10.0.0.0/16)
 371         """
 372         ret = None
 373         config.load_kube_config()
 374         api = client.CoreV1Api()
 375         kslogger = logging.getLogger('kubernetes')
 376         kslogger.setLevel(logging.INFO)
 377         res = api.list_pod_for_all_namespaces()
 378         for i in res.items:
 379             if pod in i.metadata.name:
 380                 self.logger.debug("found {0}\t{1}\t{2}".format(i.metadata.name, i.status.host_ip, i.spec.node_name))
 381                 ret = i.status.host_ip
 382                 break
 383
 384         if ret is None:
 385             ret = raw_input("Enter " + self.sdnc_controller_pod + " pod cluster node OAM IP address(10.0.0.0/16): ")
 386         return ret
 387
 388     def get_pod_node_public_ip(self, pod):
 389         """
 390         :Assuming kubectl is available and configured by default config (~/.kube/config)
 391         :param pod: pod name substring, e.g. 'sdnc-sdnc-0'
 392         :return pod's cluster node public ip (i.e. 10.12.0.0/16)
 393         """
 394         ret = None
 395         config.load_kube_config()
 396         api = client.CoreV1Api()
 397         kslogger = logging.getLogger('kubernetes')
 398         kslogger.setLevel(logging.INFO)
 399         res = api.list_pod_for_all_namespaces()
 400         for i in res.items:
 401             if pod in i.metadata.name:
 402                 ret = self.get_vm_public_ip_by_nova(i.spec.node_name)
 403                 self.logger.debug("found node {0} public ip: {1}".format(i.spec.node_name, ret))
 404                 break
 405
 406         if ret is None:
 407             ret = raw_input("Enter " + self.sdnc_controller_pod + " pod cluster node public IP address(i.e. " + self.external_net_addr + "): ")
 408         return ret
 409
 410     def get_vm_public_ip_by_nova(self, vm):
 411         """
 412         This method uses openstack nova api to retrieve vm public ip
 413         :param vm: vm name
 414         :return vm public ip
 415         """
 416         subnet = IPNetwork('{0}/{1}'.format(self.external_net_addr, self.external_net_prefix_len))
 417         nova = openstackclient.Client(2, self.cloud['--os-username'], self.cloud['--os-password'], self.cloud['--os-tenant-id'], self.cloud['--os-auth-url'])
 418         for i in nova.servers.list():
 419             if i.name == vm:
 420                 for k, v in i.networks.items():
 421                     for ip in v:
 422                         if IPAddress(ip) in subnet:
 423                             return ip
 424         return None
 425
 426     def get_vm_ip(self, keywords, net_addr=None, net_addr_len=None):
 427         """
 428         :param keywords: list of keywords to search for vm, e.g. ['bng', 'gmux', 'brg']
 429         :param net_addr: e.g. 10.12.5.0
 430         :param net_addr_len: e.g. 24
 431         :return: dictionary {keyword: ip}
 432         """
 433         if not net_addr:
 434             net_addr = self.external_net_addr
 435
 436         if not net_addr_len:
 437             net_addr_len = self.external_net_prefix_len
 438
 439         param = ' '.join([k + ' ' + v for k, v in self.cloud.items() if 'identity' not in k])
 440         openstackcmd = 'nova ' + param + ' list'
 441         self.logger.debug(openstackcmd)
 442
 443         results = os.popen(openstackcmd).read()
 444         all_vm_ip_dict = self.extract_vm_ip_as_dict(results, net_addr, net_addr_len)
 445         latest_vm_list = self.remove_old_vms(all_vm_ip_dict.keys(), self.cpe_vm_prefix)
 446         latest_vm_ip_dict = {vm: all_vm_ip_dict[vm] for vm in latest_vm_list}
 447         ip_dict = self.select_subset_vm_ip(latest_vm_ip_dict, keywords)
 448         if self.oom_mode:
 449             ip_dict.update(self.get_oom_onap_vm_ip(keywords))
 450
 451         if len(ip_dict) != len(keywords):
 452             self.logger.error('Cannot find all desired IP addresses for %s.', keywords)
 453             self.logger.error(json.dumps(ip_dict, indent=4, sort_keys=True))
 454             self.logger.error('Temporarily continue.. remember to check back vcpecommon.py line: 396')
 455 #            sys.exit()
 456         return ip_dict
 457
 458     def get_oom_onap_vm_ip(self, keywords):
 459         vm_ip = {}
 460         onap_vm_list = set(['sdc', 'so', 'sdnc', 'aai-inst1', 'robot', self.dcae_ves_collector_name])
 461         for vm in keywords:
 462             if vm in onap_vm_list:
 463                 vm_ip[vm] = self.oom_so_sdnc_aai_ip
 464         return vm_ip
 465
 466     def extract_vm_ip_as_dict(self, novalist_results, net_addr, net_addr_len):
 467         vm_ip_dict = {}
 468         for line in novalist_results.split('\n'):
 469             fields = line.split('|')
 470             if len(fields) == 8:
 471                 vm_name = fields[2]
 472                 ip_info = fields[-2]
 473                 ip = self.extract_ip_from_str(net_addr, net_addr_len, ip_info)
 474                 vm_ip_dict[vm_name] = ip
 475
 476         return vm_ip_dict
 477
 478     def remove_old_vms(self, vm_list, prefix):
 479         """
 480         For vms with format name_timestamp, only keep the one with the latest timestamp.
 481         E.g.,
 482             zdcpe1cpe01brgemu01_201805222148        (drop this)
 483             zdcpe1cpe01brgemu01_201805222229        (keep this)
 484             zdcpe1cpe01gw01_201805162201
 485         """
 486         new_vm_list = []
 487         same_type_vm_dict = {}
 488         for vm in vm_list:
 489             fields = vm.split('_')
 490             if vm.startswith(prefix) and len(fields) == 2 and len(fields[-1]) == len('201805222148') and fields[-1].isdigit():
 491                 if vm > same_type_vm_dict.get(fields[0], '0'):
 492                     same_type_vm_dict[fields[0]] = vm
 493             else:
 494                 new_vm_list.append(vm)
 495
 496         new_vm_list.extend(same_type_vm_dict.values())
 497         return new_vm_list
 498
 499     def select_subset_vm_ip(self, all_vm_ip_dict, vm_name_keyword_list):
 500         vm_ip_dict = {}
 501         for keyword in vm_name_keyword_list:
 502             for vm, ip in all_vm_ip_dict.items():
 503                 if keyword in vm:
 504                     vm_ip_dict[keyword] = ip
 505                     break
 506         return vm_ip_dict
 507
 508     def del_vgmux_ves_mode(self):
 509         url = self.vpp_ves_url.format(self.hosts['mux']) + '/mode'
 510         r = requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 511         self.logger.debug('%s', r)
 512
 513     def del_vgmux_ves_collector(self):
 514         url = self.vpp_ves_url.format(self.hosts['mux']) + '/config'
 515         r = requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 516         self.logger.debug('%s', r)
 517
 518     def set_vgmux_ves_collector(self ):
 519         url = self.vpp_ves_url.format(self.hosts['mux'])
 520         data = {'config':
 521                     {'server-addr': self.hosts[self.dcae_ves_collector_name],
 522                      'server-port': '30235' if self.oom_mode else '8081',
 523                      'read-interval': '10',
 524                      'is-add':'1'
 525                      }
 526                 }
 527         r = requests.post(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass, json=data)
 528         self.logger.debug('%s', r)
 529
 530     def set_vgmux_packet_loss_rate(self, lossrate, vg_vnf_instance_name):
 531         url = self.vpp_ves_url.format(self.hosts['mux'])
 532         data = {"mode":
 533                     {"working-mode": "demo",
 534                      "base-packet-loss": str(lossrate),
 535                      "source-name": vg_vnf_instance_name
 536                      }
 537                 }
 538         r = requests.post(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass, json=data)
 539         self.logger.debug('%s', r)
 540
 541         # return all the VxLAN interface names of BRG or vGMUX based on the IP address
 542     def get_vxlan_interfaces(self, ip, print_info=False):
 543         url = self.vpp_inf_url.format(ip)
 544         self.logger.debug('url is this: %s', url)
 545         r = requests.get(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 546         data = r.json()['interfaces']['interface']
 547         if print_info:
 548             for inf in data:
 549                 if 'name' in inf and 'type' in inf and inf['type'] == 'v3po:vxlan-tunnel':
 550                     print(json.dumps(inf, indent=4, sort_keys=True))
 551
 552         return [inf['name'] for inf in data if 'name' in inf and 'type' in inf and inf['type'] == 'v3po:vxlan-tunnel']
 553
 554     # delete all VxLAN interfaces of each hosts
 555     def delete_vxlan_interfaces(self, host_dic):
 556         for host, ip in host_dic.items():
 557             deleted = False
 558             self.logger.info('{0}: Getting VxLAN interfaces'.format(host))
 559             inf_list = self.get_vxlan_interfaces(ip)
 560             for inf in inf_list:
 561                 deleted = True
 562                 time.sleep(2)
 563                 self.logger.info("{0}: Deleting VxLAN crossconnect {1}".format(host, inf))
 564                 url = self.vpp_inf_url.format(ip) + '/interface/' + inf + '/v3po:l2'
 565                 requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 566
 567             for inf in inf_list:
 568                 deleted = True
 569                 time.sleep(2)
 570                 self.logger.info("{0}: Deleting VxLAN interface {1}".format(host, inf))
 571                 url = self.vpp_inf_url.format(ip) + '/interface/' + inf
 572                 requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 573
 574             if len(self.get_vxlan_interfaces(ip)) > 0:
 575                 self.logger.error("Error deleting VxLAN from {0}, try to restart the VM, IP is {1}.".format(host, ip))
 576                 return False
 577
 578             if not deleted:
 579                 self.logger.info("{0}: no VxLAN interface found, nothing to delete".format(host))
 580         return True
 581
 582     @staticmethod
 583     def save_object(obj, filepathname):
 584         with open(filepathname, 'wb') as fout:
 585             pickle.dump(obj, fout)
 586
 587     @staticmethod
 588     def load_object(filepathname):
 589         with open(filepathname, 'rb') as fin:
 590             return pickle.load(fin)
 591
 592     @staticmethod
 593     def increase_ip_address_or_vni_in_template(vnf_template_file, vnf_parameter_name_list):
 594         with open(vnf_template_file) as json_input:
 595             json_data = json.load(json_input)
 596             param_list = json_data['VNF-API:input']['VNF-API:vnf-topology-information']['VNF-API:vnf-parameters']
 597             for param in param_list:
 598                 if param['vnf-parameter-name'] in vnf_parameter_name_list:
 599                     ipaddr_or_vni = param['vnf-parameter-value'].split('.')
 600                     number = int(ipaddr_or_vni[-1])
 601                     if 254 == number:
 602                         number = 10
 603                     else:
 604                         number = number + 1
 605                     ipaddr_or_vni[-1] = str(number)
 606                     param['vnf-parameter-value'] = '.'.join(ipaddr_or_vni)
 607
 608         assert json_data is not None
 609         with open(vnf_template_file, 'w') as json_output:
 610             json.dump(json_data, json_output, indent=4, sort_keys=True)
 611
 612     def save_preload_data(self, preload_data):
 613         self.save_object(preload_data, self.preload_dict_file)
 614
 615     def load_preload_data(self):
 616         return self.load_object(self.preload_dict_file)
 617
 618     def save_vgmux_vnf_name(self, vgmux_vnf_name):
 619         self.save_object(vgmux_vnf_name, self.vgmux_vnf_name_file)
 620
 621     def load_vgmux_vnf_name(self):
 622         return self.load_object(self.vgmux_vnf_name_file)
 623