test/vcpe/vcpecommon.py

   1 #!/usr/bin/env python
   2
   3 import json
   4 import logging
   5 import os
   6 import pickle
   7 import re
   8 import sys
   9
  10 import ipaddress
  11 import mysql.connector
  12 import requests
  13 import commands
  14 import time
  15 import yaml
  16 from novaclient import client as openstackclient
  17 from kubernetes import client, config
  18 from netaddr import IPAddress, IPNetwork
  19
  20 class VcpeCommon:
  21
  22     def __init__(self, extra_host_names=None, cfg_file=None):
  23         self.logger = logging.getLogger(__name__)
  24         self.logger.setLevel(logging.DEBUG)
  25         self.logger.info('Initializing configuration')
  26         self.default_config = 'vcpeconfig.yaml'
  27
  28         # Read configuration from config file
  29         self._load_config(cfg_file)
  30
  31         self.sdnc_controller_pod = '-'.join([self.onap_environment, 'sdnc-sdnc-0'])
  32         # OOM: this is the address that the brg and bng will nat for sdnc access - 10.0.0.x address of k8 host for sdnc-0 container
  33         self.sdnc_oam_ip = self.get_pod_node_oam_ip(self.sdnc_controller_pod)
  34         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
  35         self.oom_so_sdnc_aai_ip = self.get_pod_node_public_ip(self.sdnc_controller_pod)
  36         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
  37         self.oom_dcae_ves_collector = self.oom_so_sdnc_aai_ip
  38         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
  39         self.mr_ip_addr = self.oom_so_sdnc_aai_ip
  40         self.mr_ip_port = '30227'
  41         self.so_nbi_port = '30277' if self.oom_mode else '8080'
  42         self.sdnc_preloading_port = '30267' if self.oom_mode else '8282'
  43         self.aai_query_port = '30233' if self.oom_mode else '8443'
  44         self.sniro_port = '30288' if self.oom_mode else '8080'
  45
  46         self.host_names = ['sdc', 'so', 'sdnc', 'robot', 'aai-inst1', self.dcae_ves_collector_name, 'mariadb-galera']
  47         if extra_host_names:
  48             self.host_names.extend(extra_host_names)
  49         # get IP addresses
  50         self.hosts = self.get_vm_ip(self.host_names, self.external_net_addr, self.external_net_prefix_len)
  51         # this is the keyword used to name vgw stack, must not be used in other stacks
  52         self.vgw_name_keyword = 'base_vcpe_vgw'
  53         # this is the file that will keep the index of last assigned SO name
  54         self.vgw_vfmod_name_index_file= '__var/vgw_vfmod_name_index'
  55         self.svc_instance_uuid_file = '__var/svc_instance_uuid'
  56         self.preload_dict_file = '__var/preload_dict'
  57         self.vgmux_vnf_name_file = '__var/vgmux_vnf_name'
  58         self.product_family_id = 'f9457e8c-4afd-45da-9389-46acd9bf5116'
  59         self.custom_product_family_id = 'a9a77d5a-123e-4ca2-9eb9-0b015d2ee0fb'
  60         self.instance_name_prefix = {
  61             'service': 'vcpe_svc',
  62             'network': 'vcpe_net',
  63             'vnf': 'vcpe_vnf',
  64             'vfmodule': 'vcpe_vfmodule'
  65         }
  66         self.aai_userpass = 'AAI', 'AAI'
  67         self.os_tenant_id = self.cloud['--os-tenant-id']
  68         self.os_region_name = self.cloud['--os-region-name']
  69         self.common_preload_config['pub_key'] = self.pub_key
  70         self.sniro_url = 'http://' + self.hosts['robot'] + ':' + self.sniro_port + '/__admin/mappings'
  71         self.sniro_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
  72         self.homing_solution = 'sniro'  # value is either 'sniro' or 'oof'
  73 #        self.homing_solution = 'oof'
  74         self.customer_location_used_by_oof = {
  75             "customerLatitude": "32.897480",
  76             "customerLongitude": "-97.040443",
  77             "customerName": "some_company"
  78         }
  79
  80         #############################################################################################
  81         # SDC urls
  82         self.sdc_be_port = '30204'
  83         self.sdc_be_request_userpass = 'vid', 'Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U'
  84         self.sdc_be_request_headers = {'X-ECOMP-InstanceID': 'VID'}
  85         self.sdc_be_url_prefix = 'https://' + self.hosts['sdc'] + ':' + self.sdc_be_port
  86         self.sdc_service_list_url = self.sdc_be_url_prefix + '/sdc/v1/catalog/services'
  87
  88         self.sdc_fe_port = '30207'
  89         self.sdc_fe_request_userpass = 'beep', 'boop'
  90         self.sdc_fe_request_headers = {'USER_ID': 'demo', 'Content-Type': 'application/json'}
  91         self.sdc_fe_url_prefix = 'https://' + self.hosts['sdc'] + ':' + self.sdc_fe_port
  92         self.sdc_get_category_list_url = self.sdc_fe_url_prefix + '/sdc1/feProxy/rest/v1/categories'
  93         self.sdc_create_allotted_resource_subcategory_url = self.sdc_fe_url_prefix + '/sdc1/feProxy/rest/v1/category/resources/resourceNewCategory.allotted%20resource/subCategory'
  94
  95         #############################################################################################
  96         # SDNC urls
  97         self.sdnc_userpass = 'admin', 'Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U'
  98         self.sdnc_db_name = 'sdnctl'
  99         self.sdnc_db_user = 'sdnctl'
 100         self.sdnc_db_pass = 'gamma'
 101         self.sdnc_db_port = self.get_k8s_service_endpoint_info('mariadb-galera','port') if self.oom_mode else '3306'
 102         self.sdnc_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 103         self.sdnc_preload_network_url = 'https://' + self.hosts['sdnc'] + \
 104                                         ':' + self.sdnc_preloading_port + '/restconf/operations/VNF-API:preload-network-topology-operation'
 105         self.sdnc_preload_network_gra_url = 'https://' + self.hosts['sdnc'] + \
 106                                         ':' + self.sdnc_preloading_port + '/restconf/operations/GENERIC-RESOURCE-API:preload-network-topology-operation'
 107         self.sdnc_preload_vnf_url = 'https://' + self.hosts['sdnc'] + \
 108                                     ':' + self.sdnc_preloading_port + '/restconf/operations/VNF-API:preload-vnf-topology-operation'
 109         self.sdnc_preload_gra_url = 'https://' + self.hosts['sdnc'] + \
 110                                     ':' + self.sdnc_preloading_port + '/restconf/operations/GENERIC-RESOURCE-API:preload-vf-module-topology-operation'
 111         self.sdnc_ar_cleanup_url = 'https://' + self.hosts['sdnc'] + ':' + self.sdnc_preloading_port + \
 112                                    '/restconf/config/GENERIC-RESOURCE-API:'
 113
 114         #############################################################################################
 115         # MARIADB-GALERA settings
 116         self.mariadb_galera_endpoint_ip = self.get_k8s_service_endpoint_info('mariadb-galera','ip')
 117         self.mariadb_galera_endpoint_port = self.get_k8s_service_endpoint_info('mariadb-galera','port')
 118
 119         #############################################################################################
 120         # SO urls, note: do NOT add a '/' at the end of the url
 121         self.so_req_api_url = {'v4': 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/serviceInstantiation/v7/serviceInstances',
 122                            'v5': 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/serviceInstantiation/v7/serviceInstances'}
 123         self.so_check_progress_api_url = 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/orchestrationRequests/v6'
 124         self.so_userpass = 'InfraPortalClient', 'password1$'
 125         self.so_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 126         self.so_db_name = 'catalogdb'
 127         self.so_db_user = 'root'
 128         self.so_db_pass = 'secretpassword'
 129         self.so_db_host = self.mariadb_galera_endpoint_ip if self.oom_mode else self.hosts['so']
 130         self.so_db_port = self.mariadb_galera_endpoint_port if self.oom_mode else '3306'
 131
 132         self.vpp_inf_url = 'http://{0}:8183/restconf/config/ietf-interfaces:interfaces'
 133         self.vpp_api_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 134         self.vpp_api_userpass = ('admin', 'admin')
 135         self.vpp_ves_url= 'http://{0}:8183/restconf/config/vesagent:vesagent'
 136
 137         #############################################################################################
 138         # POLICY urls
 139         self.policy_userpass = ('healthcheck', 'zb!XztG34')
 140         self.policy_headers = {'Accept': 'application/json', 'Content-Type': 'application/json'}
 141         self.policy_api_url = 'https://{0}:6969/policy/api/v1/policytypes/onap.policies.controlloop.Operational/versions/1.0.0/policies'
 142         self.policy_pap_get_url = 'https://{0}:6969/policy/pap/v1/pdps'
 143         self.policy_pap_json = {'policies': [{'policy-id': 'operational.vcpe'}]}
 144         self.policy_pap_post_url = self.policy_pap_get_url + '/policies'
 145         self.policy_api_service_name = 'policy-api'
 146         self.policy_pap_service_name = 'policy-pap'
 147
 148         #############################################################################################
 149         # AAI urls
 150         self.aai_region_query_url = 'https://' + self.oom_so_sdnc_aai_ip + ':' +\
 151                                     self.aai_query_port +\
 152                                     '/aai/v14/cloud-infrastructure/cloud-regions/cloud-region/CloudOwner/' +\
 153                                     self.cloud['--os-region-name']
 154         self.aai_headers = {'Accept': 'application/json',
 155                             'Content-Type': 'application/json',
 156                             'X-FromAppId': 'postman', 'X-TransactionId': '9999'}
 157
 158     def _load_config(self, cfg_file):
 159         """
 160         Reads vcpe config file and injects settings as object's attributes
 161         :param cfg_file: Configuration file path
 162         """
 163
 164         if cfg_file is None:
 165             cfg_file = self.default_config
 166
 167         try:
 168             with open(cfg_file, 'r') as cfg:
 169                 cfg_yml = yaml.full_load(cfg)
 170         except Exception as e:
 171             self.logger.error('Error loading configuration: ' + str(e))
 172             sys.exit(1)
 173
 174         self.logger.debug('\n' + yaml.dump(cfg_yml))
 175
 176         # Use setattr to load config file keys as VcpeCommon class' object
 177         # attributes
 178         try:
 179             # Check config isn't empty
 180             if cfg_yml is not None:
 181                 for cfg_key in cfg_yml:
 182                     setattr(self, cfg_key, cfg_yml[cfg_key])
 183         except TypeError as e:
 184             self.logger.error('Unable to parse config file: ' + str(e))
 185             sys.exit(1)
 186
 187     def heatbridge(self, openstack_stack_name, svc_instance_uuid):
 188         """
 189         Add vserver information to AAI
 190         """
 191         self.logger.info('Adding vServer information to AAI for {0}'.format(openstack_stack_name))
 192         if not self.oom_mode:
 193             cmd = '/opt/demo.sh heatbridge {0} {1} vCPE'.format(openstack_stack_name, svc_instance_uuid)
 194             ret = commands.getstatusoutput("ssh -i onap_dev root@{0} '{1}'".format(self.hosts['robot'], cmd))
 195             self.logger.debug('%s', ret)
 196         else:
 197             print('To add vGMUX vserver info to AAI, do the following:')
 198             print('- ssh to rancher')
 199             print('- sudo su -')
 200             print('- cd /root/oom/kubernetes/robot')
 201             print('- ./demo-k8s.sh onap heatbridge {0} {1} vCPE'.format(openstack_stack_name, svc_instance_uuid))
 202
 203     def get_brg_mac_from_sdnc(self):
 204         """
 205         Check table DHCP_MAP in the SDNC DB. Find the newly instantiated BRG MAC address.
 206         Note that there might be multiple BRGs, the most recently instantiated BRG always has the largest IP address.
 207         """
 208         if self.oom_mode:
 209             db_host=self.mariadb_galera_endpoint_ip
 210         else:
 211             db_host=self.hosts['mariadb-galera']
 212
 213         cnx = mysql.connector.connect(user=self.sdnc_db_user,
 214                                       password=self.sdnc_db_pass,
 215                                       database=self.sdnc_db_name,
 216                                       host=db_host,
 217                                       port=self.sdnc_db_port)
 218         cursor = cnx.cursor()
 219         query = "SELECT * from DHCP_MAP"
 220         cursor.execute(query)
 221
 222         self.logger.debug('DHCP_MAP table in SDNC')
 223         mac_recent = None
 224         host = -1
 225         for mac, ip in cursor:
 226             self.logger.debug(mac + ' - ' + ip)
 227             this_host = int(ip.split('.')[-1])
 228             if host < this_host:
 229                 host = this_host
 230                 mac_recent = mac
 231
 232         cnx.close()
 233
 234         try:
 235             assert mac_recent
 236         except AssertionError:
 237             self.logger.error('Failed to obtain BRG MAC address from database')
 238             sys.exit(1)
 239
 240         return mac_recent
 241
 242     def execute_cmds_mariadb(self, cmds):
 243         self.execute_cmds_db(cmds, self.sdnc_db_user, self.sdnc_db_pass,
 244                              self.sdnc_db_name, self.mariadb_galera_endpoint_ip,
 245                              self.mariadb_galera_endpoint_port)
 246
 247     def execute_cmds_sdnc_db(self, cmds):
 248         self.execute_cmds_db(cmds, self.sdnc_db_user, self.sdnc_db_pass, self.sdnc_db_name,
 249                              self.hosts['sdnc'], self.sdnc_db_port)
 250
 251     def execute_cmds_so_db(self, cmds):
 252         self.execute_cmds_db(cmds, self.so_db_user, self.so_db_pass, self.so_db_name,
 253                              self.so_db_host, self.so_db_port)
 254
 255     def execute_cmds_db(self, cmds, dbuser, dbpass, dbname, host, port):
 256         cnx = mysql.connector.connect(user=dbuser, password=dbpass, database=dbname, host=host, port=port)
 257         cursor = cnx.cursor()
 258         for cmd in cmds:
 259             self.logger.debug(cmd)
 260             cursor.execute(cmd)
 261             self.logger.debug('%s', cursor)
 262         cnx.commit()
 263         cursor.close()
 264         cnx.close()
 265
 266     def find_file(self, file_name_keyword, file_ext, search_dir):
 267         """
 268         :param file_name_keyword:  keyword used to look for the csar file, case insensitive matching, e.g, infra
 269         :param file_ext: e.g., csar, json
 270         :param search_dir path to search
 271         :return: path name of the file
 272         """
 273         file_name_keyword = file_name_keyword.lower()
 274         file_ext = file_ext.lower()
 275         if not file_ext.startswith('.'):
 276             file_ext = '.' + file_ext
 277
 278         filenamepath = None
 279         for file_name in os.listdir(search_dir):
 280             file_name_lower = file_name.lower()
 281             if file_name_keyword in file_name_lower and file_name_lower.endswith(file_ext):
 282                 if filenamepath:
 283                     self.logger.error('Multiple files found for *{0}*.{1} in '
 284                                       'directory {2}'.format(file_name_keyword, file_ext, search_dir))
 285                     sys.exit(1)
 286                 filenamepath = os.path.abspath(os.path.join(search_dir, file_name))
 287
 288         if filenamepath:
 289             return filenamepath
 290         else:
 291             self.logger.error("Cannot find *{0}*{1} in directory {2}".format(file_name_keyword, file_ext, search_dir))
 292             sys.exit(1)
 293
 294     @staticmethod
 295     def network_name_to_subnet_name(network_name):
 296         """
 297         :param network_name: example: vcpe_net_cpe_signal_201711281221
 298         :return: vcpe_net_cpe_signal_subnet_201711281221
 299         """
 300         fields = network_name.split('_')
 301         fields.insert(-1, 'subnet')
 302         return '_'.join(fields)
 303
 304     def set_network_name(self, network_name):
 305         param = ' '.join([k + ' ' + v for k, v in self.cloud.items()])
 306         openstackcmd = 'openstack ' + param
 307         cmd = ' '.join([openstackcmd, 'network set --name', network_name, 'ONAP-NW1'])
 308         os.popen(cmd)
 309
 310     def set_subnet_name(self, network_name):
 311         """
 312         Example: network_name =  vcpe_net_cpe_signal_201711281221
 313         set subnet name to vcpe_net_cpe_signal_subnet_201711281221
 314         :return:
 315         """
 316         param = ' '.join([k + ' ' + v for k, v in self.cloud.items()])
 317         openstackcmd = 'openstack ' + param
 318
 319         # expected results: | subnets | subnet_id |
 320         subnet_info = os.popen(openstackcmd + ' network show ' + network_name + ' |grep subnets').read().split('|')
 321         if len(subnet_info) > 2 and subnet_info[1].strip() == 'subnets':
 322             subnet_id = subnet_info[2].strip()
 323             subnet_name = self.network_name_to_subnet_name(network_name)
 324             cmd = ' '.join([openstackcmd, 'subnet set --name', subnet_name, subnet_id])
 325             os.popen(cmd)
 326             self.logger.info("Subnet name set to: " + subnet_name)
 327             return True
 328         else:
 329             self.logger.error("Can't get subnet info from network name: " + network_name)
 330             return False
 331
 332     def set_closed_loop_policy(self, policy_template_file):
 333         # Gather policy services cluster ips
 334         p_api_cluster_ip = self.get_k8s_service_cluster_ip(self.policy_api_service_name)
 335         p_pap_cluster_ip = self.get_k8s_service_cluster_ip(self.policy_pap_service_name)
 336
 337         # Read policy json from file
 338         with open(policy_template_file) as f:
 339             try:
 340                 policy_json = json.load(f)
 341             except ValueError:
 342                 self.logger.error(policy_template_file + " doesn't seem to contain valid JSON data")
 343                 sys.exit(1)
 344
 345         # Check policy already applied
 346         policy_exists_req = requests.get(self.policy_pap_get_url.format(
 347                             p_pap_cluster_ip), auth=self.policy_userpass,
 348                             verify=False, headers=self.policy_headers)
 349         if policy_exists_req.status_code != 200:
 350             self.logger.error('Failure in checking CL policy existence. '
 351                                'Policy-pap responded with HTTP code {0}'.format(
 352                                policy_exists_req.status_code))
 353             sys.exit(1)
 354
 355         try:
 356             policy_exists_json = policy_exists_req.json()
 357         except ValueError as e:
 358             self.logger.error('Policy-pap request failed: ' + e.message)
 359             sys.exit(1)
 360
 361         try:
 362             assert policy_exists_json['groups'][0]['pdpSubgroups'] \
 363                                [1]['policies'][0]['name'] != 'operational.vcpe'
 364         except AssertionError:
 365             self.logger.info('vCPE closed loop policy already exists, not applying')
 366             return
 367         except IndexError:
 368             pass # policy doesn't exist
 369
 370         # Create policy
 371         policy_create_req = requests.post(self.policy_api_url.format(
 372                             p_api_cluster_ip), auth=self.policy_userpass,
 373                             json=policy_json, verify=False,
 374                             headers=self.policy_headers)
 375         # Get the policy id from policy-api response
 376         if policy_create_req.status_code != 200:
 377             self.logger.error('Failed creating policy. Policy-api responded'
 378                               ' with HTTP code {0}'.format(policy_create_req.status_code))
 379             sys.exit(1)
 380
 381         try:
 382             policy_version = json.loads(policy_create_req.text)['policy-version']
 383         except (KeyError, ValueError):
 384             self.logger.error('Policy API response not understood:')
 385             self.logger.debug('\n' + str(policy_create_req.text))
 386
 387         # Inject the policy into Policy PAP
 388         self.policy_pap_json['policies'].append({'policy-version': policy_version})
 389         policy_insert_req = requests.post(self.policy_pap_post_url.format(
 390                             p_pap_cluster_ip), auth=self.policy_userpass,
 391                             json=self.policy_pap_json, verify=False,
 392                             headers=self.policy_headers)
 393         if policy_insert_req.status_code != 200:
 394             self.logger.error('Policy PAP request failed with HTTP code'
 395                               '{0}'.format(policy_insert_req.status_code))
 396             sys.exit(1)
 397         self.logger.info('Successully pushed closed loop Policy')
 398
 399     def is_node_in_aai(self, node_type, node_uuid):
 400         key = None
 401         search_node_type = None
 402         if node_type == 'service':
 403             search_node_type = 'service-instance'
 404             key = 'service-instance-id'
 405         elif node_type == 'vnf':
 406             search_node_type = 'generic-vnf'
 407             key = 'vnf-id'
 408         else:
 409             logging.error('Invalid node_type: ' + node_type)
 410             sys.exit(1)
 411
 412         url = 'https://{0}:{1}/aai/v11/search/nodes-query?search-node-type={2}&filter={3}:EQUALS:{4}'.format(
 413             self.hosts['aai-inst1'], self.aai_query_port, search_node_type, key, node_uuid)
 414
 415         headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'X-FromAppID': 'vCPE-Robot', 'X-TransactionId': 'get_aai_subscr'}
 416         r = requests.get(url, headers=headers, auth=self.aai_userpass, verify=False)
 417         response = r.json()
 418         self.logger.debug('aai query: ' + url)
 419         self.logger.debug('aai response:\n' + json.dumps(response, indent=4, sort_keys=True))
 420         return 'result-data' in response
 421
 422     @staticmethod
 423     def extract_ip_from_str(net_addr, net_addr_len, sz):
 424         """
 425         :param net_addr:  e.g. 10.5.12.0
 426         :param net_addr_len: e.g. 24
 427         :param sz: a string
 428         :return: the first IP address matching the network, e.g. 10.5.12.3
 429         """
 430         network = ipaddress.ip_network(unicode('{0}/{1}'.format(net_addr, net_addr_len)), strict=False)
 431         ip_list = re.findall(r'[0-9]+(?:\.[0-9]+){3}', sz)
 432         for ip in ip_list:
 433             this_net = ipaddress.ip_network(unicode('{0}/{1}'.format(ip, net_addr_len)), strict=False)
 434             if this_net == network:
 435                 return str(ip)
 436         return None
 437
 438     def get_pod_node_oam_ip(self, pod):
 439         """
 440         :Assuming kubectl is available and configured by default config (~/.kube/config)
 441         :param pod: pod name substring, e.g. 'sdnc-sdnc-0'
 442         :return pod's cluster node oam ip (10.0.0.0/16)
 443         """
 444         ret = None
 445         config.load_kube_config()
 446         api = client.CoreV1Api()
 447         kslogger = logging.getLogger('kubernetes')
 448         kslogger.setLevel(logging.INFO)
 449         res = api.list_pod_for_all_namespaces()
 450         for i in res.items:
 451             if pod in i.metadata.name:
 452                 self.logger.debug("found {0}\t{1}\t{2}".format(i.metadata.name, i.status.host_ip, i.spec.node_name))
 453                 ret = i.status.host_ip
 454                 break
 455
 456         if ret is None:
 457             ret = raw_input("Enter " + self.sdnc_controller_pod + " pod cluster node OAM IP address(10.0.0.0/16): ")
 458         return ret
 459
 460     def get_pod_node_public_ip(self, pod):
 461         """
 462         :Assuming kubectl is available and configured by default config (~/.kube/config)
 463         :param pod: pod name substring, e.g. 'sdnc-sdnc-0'
 464         :return pod's cluster node public ip (i.e. 10.12.0.0/16)
 465         """
 466         ret = None
 467         config.load_kube_config()
 468         api = client.CoreV1Api()
 469         kslogger = logging.getLogger('kubernetes')
 470         kslogger.setLevel(logging.INFO)
 471         res = api.list_pod_for_all_namespaces()
 472         for i in res.items:
 473             if pod in i.metadata.name:
 474                 ret = self.get_vm_public_ip_by_nova(i.spec.node_name)
 475                 self.logger.debug("found node {0} public ip: {1}".format(i.spec.node_name, ret))
 476                 break
 477
 478         if ret is None:
 479             ret = raw_input("Enter " + self.sdnc_controller_pod + " pod cluster node public IP address(i.e. " + self.external_net_addr + "): ")
 480         return ret
 481
 482     def get_vm_public_ip_by_nova(self, vm):
 483         """
 484         This method uses openstack nova api to retrieve vm public ip
 485         :param vm: vm name
 486         :return vm public ip
 487         """
 488         subnet = IPNetwork('{0}/{1}'.format(self.external_net_addr, self.external_net_prefix_len))
 489         nova = openstackclient.Client(2, self.cloud['--os-username'], self.cloud['--os-password'], self.cloud['--os-tenant-id'], self.cloud['--os-auth-url'])
 490         for i in nova.servers.list():
 491             if i.name == vm:
 492                 for k, v in i.networks.items():
 493                     for ip in v:
 494                         if IPAddress(ip) in subnet:
 495                             return ip
 496         return None
 497
 498     def get_vm_ip(self, keywords, net_addr=None, net_addr_len=None):
 499         """
 500         :param keywords: list of keywords to search for vm, e.g. ['bng', 'gmux', 'brg']
 501         :param net_addr: e.g. 10.12.5.0
 502         :param net_addr_len: e.g. 24
 503         :return: dictionary {keyword: ip}
 504         """
 505         if not net_addr:
 506             net_addr = self.external_net_addr
 507
 508         if not net_addr_len:
 509             net_addr_len = self.external_net_prefix_len
 510
 511         param = ' '.join([k + ' ' + v for k, v in self.cloud.items() if 'identity' not in k])
 512         openstackcmd = 'nova ' + param + ' list'
 513         self.logger.debug(openstackcmd)
 514
 515         results = os.popen(openstackcmd).read()
 516         all_vm_ip_dict = self.extract_vm_ip_as_dict(results, net_addr, net_addr_len)
 517         latest_vm_list = self.remove_old_vms(all_vm_ip_dict.keys(), self.cpe_vm_prefix)
 518         latest_vm_ip_dict = {vm: all_vm_ip_dict[vm] for vm in latest_vm_list}
 519         ip_dict = self.select_subset_vm_ip(latest_vm_ip_dict, keywords)
 520         if self.oom_mode:
 521             ip_dict.update(self.get_oom_onap_vm_ip(keywords))
 522
 523         if len(ip_dict) != len(keywords):
 524             self.logger.error('Cannot find all desired IP addresses for %s.', keywords)
 525             self.logger.error(json.dumps(ip_dict, indent=4, sort_keys=True))
 526             self.logger.error('Temporarily continue.. remember to check back vcpecommon.py line: 396')
 527 #            sys.exit(1)
 528         return ip_dict
 529
 530     def get_oom_onap_vm_ip(self, keywords):
 531         vm_ip = {}
 532         for vm in keywords:
 533             if vm in self.host_names:
 534                 vm_ip[vm] = self.oom_so_sdnc_aai_ip
 535         return vm_ip
 536
 537     def get_k8s_service_cluster_ip(self, service):
 538         """
 539         Returns cluster IP for a given service
 540         :param service: name of the service
 541         :return: cluster ip
 542         """
 543         config.load_kube_config()
 544         api = client.CoreV1Api()
 545         kslogger = logging.getLogger('kubernetes')
 546         kslogger.setLevel(logging.INFO)
 547         try:
 548             resp = api.read_namespaced_service(service, self.onap_namespace)
 549         except client.rest.ApiException as e:
 550             self.logger.error('Error while making k8s API request: ' + e.body)
 551             sys.exit(1)
 552
 553         return resp.spec.cluster_ip
 554
 555     def get_k8s_service_endpoint_info(self, service, subset):
 556         """
 557         Returns endpoint data for a given service and subset. If there
 558         is more than one endpoint returns data for the first one from
 559         the list that API returned.
 560         :param service: name of the service
 561         :param subset: subset name, one of "ip","port"
 562         :return: endpoint ip
 563         """
 564         config.load_kube_config()
 565         api = client.CoreV1Api()
 566         kslogger = logging.getLogger('kubernetes')
 567         kslogger.setLevel(logging.INFO)
 568         try:
 569             resp = api.read_namespaced_endpoints(service, self.onap_namespace)
 570         except client.rest.ApiException as e:
 571             self.logger.error('Error while making k8s API request: ' + e.body)
 572             sys.exit(1)
 573
 574         if subset == "ip":
 575             return resp.subsets[0].addresses[0].ip
 576         elif subset == "port":
 577             return resp.subsets[0].ports[0].port
 578         else:
 579             self.logger.error("Unsupported subset type")
 580
 581     def extract_vm_ip_as_dict(self, novalist_results, net_addr, net_addr_len):
 582         vm_ip_dict = {}
 583         for line in novalist_results.split('\n'):
 584             fields = line.split('|')
 585             if len(fields) == 8:
 586                 vm_name = fields[2]
 587                 ip_info = fields[-2]
 588                 ip = self.extract_ip_from_str(net_addr, net_addr_len, ip_info)
 589                 vm_ip_dict[vm_name] = ip
 590
 591         return vm_ip_dict
 592
 593     def remove_old_vms(self, vm_list, prefix):
 594         """
 595         For vms with format name_timestamp, only keep the one with the latest timestamp.
 596         E.g.,
 597             zdcpe1cpe01brgemu01_201805222148        (drop this)
 598             zdcpe1cpe01brgemu01_201805222229        (keep this)
 599             zdcpe1cpe01gw01_201805162201
 600         """
 601         new_vm_list = []
 602         same_type_vm_dict = {}
 603         for vm in vm_list:
 604             fields = vm.split('_')
 605             if vm.startswith(prefix) and len(fields) == 2 and len(fields[-1]) == len('201805222148') and fields[-1].isdigit():
 606                 if vm > same_type_vm_dict.get(fields[0], '0'):
 607                     same_type_vm_dict[fields[0]] = vm
 608             else:
 609                 new_vm_list.append(vm)
 610
 611         new_vm_list.extend(same_type_vm_dict.values())
 612         return new_vm_list
 613
 614     def select_subset_vm_ip(self, all_vm_ip_dict, vm_name_keyword_list):
 615         vm_ip_dict = {}
 616         for keyword in vm_name_keyword_list:
 617             for vm, ip in all_vm_ip_dict.items():
 618                 if keyword in vm:
 619                     vm_ip_dict[keyword] = ip
 620                     break
 621         return vm_ip_dict
 622
 623     def del_vgmux_ves_mode(self):
 624         url = self.vpp_ves_url.format(self.hosts['mux']) + '/mode'
 625         r = requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 626         self.logger.debug('%s', r)
 627
 628     def del_vgmux_ves_collector(self):
 629         url = self.vpp_ves_url.format(self.hosts['mux']) + '/config'
 630         r = requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 631         self.logger.debug('%s', r)
 632
 633     def set_vgmux_ves_collector(self ):
 634         url = self.vpp_ves_url.format(self.hosts['mux'])
 635         data = {'config':
 636                     {'server-addr': self.hosts[self.dcae_ves_collector_name],
 637                      'server-port': '30235' if self.oom_mode else '8081',
 638                      'read-interval': '10',
 639                      'is-add':'1'
 640                      }
 641                 }
 642         r = requests.post(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass, json=data)
 643         self.logger.debug('%s', r)
 644
 645     def set_vgmux_packet_loss_rate(self, lossrate, vg_vnf_instance_name):
 646         url = self.vpp_ves_url.format(self.hosts['mux'])
 647         data = {"mode":
 648                     {"working-mode": "demo",
 649                      "base-packet-loss": str(lossrate),
 650                      "source-name": vg_vnf_instance_name
 651                      }
 652                 }
 653         r = requests.post(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass, json=data)
 654         self.logger.debug('%s', r)
 655
 656         # return all the VxLAN interface names of BRG or vGMUX based on the IP address
 657     def get_vxlan_interfaces(self, ip, print_info=False):
 658         url = self.vpp_inf_url.format(ip)
 659         self.logger.debug('url is this: %s', url)
 660         r = requests.get(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 661         data = r.json()['interfaces']['interface']
 662         if print_info:
 663             for inf in data:
 664                 if 'name' in inf and 'type' in inf and inf['type'] == 'v3po:vxlan-tunnel':
 665                     print(json.dumps(inf, indent=4, sort_keys=True))
 666
 667         return [inf['name'] for inf in data if 'name' in inf and 'type' in inf and inf['type'] == 'v3po:vxlan-tunnel']
 668
 669     # delete all VxLAN interfaces of each hosts
 670     def delete_vxlan_interfaces(self, host_dic):
 671         for host, ip in host_dic.items():
 672             deleted = False
 673             self.logger.info('{0}: Getting VxLAN interfaces'.format(host))
 674             inf_list = self.get_vxlan_interfaces(ip)
 675             for inf in inf_list:
 676                 deleted = True
 677                 time.sleep(2)
 678                 self.logger.info("{0}: Deleting VxLAN crossconnect {1}".format(host, inf))
 679                 url = self.vpp_inf_url.format(ip) + '/interface/' + inf + '/v3po:l2'
 680                 requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 681
 682             for inf in inf_list:
 683                 deleted = True
 684                 time.sleep(2)
 685                 self.logger.info("{0}: Deleting VxLAN interface {1}".format(host, inf))
 686                 url = self.vpp_inf_url.format(ip) + '/interface/' + inf
 687                 requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 688
 689             if len(self.get_vxlan_interfaces(ip)) > 0:
 690                 self.logger.error("Error deleting VxLAN from {0}, try to restart the VM, IP is {1}.".format(host, ip))
 691                 return False
 692
 693             if not deleted:
 694                 self.logger.info("{0}: no VxLAN interface found, nothing to delete".format(host))
 695         return True
 696
 697     @staticmethod
 698     def save_object(obj, filepathname):
 699         with open(filepathname, 'wb') as fout:
 700             pickle.dump(obj, fout)
 701
 702     @staticmethod
 703     def load_object(filepathname):
 704         with open(filepathname, 'rb') as fin:
 705             return pickle.load(fin)
 706
 707     @staticmethod
 708     def increase_ip_address_or_vni_in_template(vnf_template_file, vnf_parameter_name_list):
 709         with open(vnf_template_file) as json_input:
 710             json_data = json.load(json_input)
 711             param_list = json_data['VNF-API:input']['VNF-API:vnf-topology-information']['VNF-API:vnf-parameters']
 712             for param in param_list:
 713                 if param['vnf-parameter-name'] in vnf_parameter_name_list:
 714                     ipaddr_or_vni = param['vnf-parameter-value'].split('.')
 715                     number = int(ipaddr_or_vni[-1])
 716                     if 254 == number:
 717                         number = 10
 718                     else:
 719                         number = number + 1
 720                     ipaddr_or_vni[-1] = str(number)
 721                     param['vnf-parameter-value'] = '.'.join(ipaddr_or_vni)
 722
 723         assert json_data is not None
 724         with open(vnf_template_file, 'w') as json_output:
 725             json.dump(json_data, json_output, indent=4, sort_keys=True)
 726
 727     def save_preload_data(self, preload_data):
 728         self.save_object(preload_data, self.preload_dict_file)
 729
 730     def load_preload_data(self):
 731         return self.load_object(self.preload_dict_file)
 732
 733     def save_vgmux_vnf_name(self, vgmux_vnf_name):
 734         self.save_object(vgmux_vnf_name, self.vgmux_vnf_name_file)
 735
 736     def load_vgmux_vnf_name(self):
 737         return self.load_object(self.vgmux_vnf_name_file)
 738