test/vcpe/vcpecommon.py

   1 #!/usr/bin/env python
   2
   3 import json
   4 import logging
   5 import os
   6 import pickle
   7 import re
   8 import sys
   9
  10 import ipaddress
  11 import mysql.connector
  12 import requests
  13 import commands
  14 import time
  15 from novaclient import client as openstackclient
  16 from kubernetes import client, config
  17 from netaddr import IPAddress, IPNetwork
  18
  19 ######################################################################
  20 # Parts which must be updated / cross-checked during each deployment #
  21 # are marked as CHANGEME                                             #
  22 ######################################################################
  23
  24 class VcpeCommon:
  25     #############################################################################################
  26     # Set network prefix of k8s host external address; it's used for pod public IP autodetection
  27     # but can be overriden from user in case of autodetection failure
  28     external_net_addr = '10.12.0.0'
  29     external_net_prefix_len = 16
  30
  31     #############################################################################################
  32     # set the openstack cloud access credentials here
  33     oom_mode = True
  34
  35     ###########################
  36     # set Openstack credentials
  37     # CHANGEME part
  38     cloud = {
  39         '--os-auth-url': 'http://10.12.25.2:5000',
  40         '--os-username': 'kxi',
  41         '--os-user-domain-id': 'default',
  42         '--os-project-domain-id': 'default',
  43         '--os-tenant-id': 'bc43d50ffcb84750bac0c1707a9a765b' if oom_mode else '1e097c6713e74fd7ac8e4295e605ee1e',
  44         '--os-region-name': 'RegionOne',
  45         '--os-password': 'n3JhGMGuDzD8',
  46         '--os-project-domain-name': 'Integration-SB-03' if oom_mode else 'Integration-SB-07',
  47         '--os-identity-api-version': '3'
  48     }
  49
  50     ############################################################################
  51     # set oam and public network which must exist in openstack before deployment
  52     # CHANGEME part
  53     common_preload_config = {
  54         'oam_onap_net': 'oam_network_2No2' if oom_mode else 'oam_onap_lAky',
  55         'oam_onap_subnet': 'oam_network_2No2' if oom_mode else 'oam_onap_lAky',
  56         'public_net': 'external',
  57         'public_net_id': '971040b2-7059-49dc-b220-4fab50cb2ad4'
  58     }
  59
  60     #############################################################################
  61     # Set name of Onap's k8s namespace and sdnc controller pod
  62     # CHANGEME part
  63     onap_namespace = 'dev'
  64     sdnc_controller_pod = '-'.join([onap_namespace,'sdnc-sdnc-0'])
  65
  66     template_variable_symbol = '${'
  67     cpe_vm_prefix = 'zdcpe'
  68
  69     #############################################################################################
  70     # preloading network config
  71     #  key=network role
  72     #  value = [subnet_start_ip, subnet_gateway_ip]
  73     preload_network_config = {
  74         'cpe_public': ['10.2.0.2', '10.2.0.1'],
  75         'cpe_signal': ['10.4.0.2', '10.4.0.1'],
  76         'brg_bng': ['10.3.0.2', '10.3.0.1'],
  77         'bng_mux': ['10.1.0.10', '10.1.0.1'],
  78         'mux_gw': ['10.5.0.10', '10.5.0.1']
  79     }
  80
  81     dcae_ves_collector_name = 'dcae-bootstrap'
  82     global_subscriber_id = 'SDN-ETHERNET-INTERNET'
  83     project_name = 'Project-Demonstration'
  84     owning_entity_id = '520cc603-a3c4-4ec2-9ef4-ca70facd79c0'
  85     owning_entity_name = 'OE-Demonstration1'
  86
  87     def __init__(self, extra_host_names=None):
  88         self.logger = logging.getLogger(__name__)
  89         self.logger.setLevel(logging.DEBUG)
  90         self.logger.info('Initializing configuration')
  91
  92         ##################################################################################################################################
  93         # following param must be updated e.g. from csar file (grep for VfModuleModelInvariantUuid string) before vcpe.py customer call !!
  94         # vgw_VfModuleModelInvariantUuid is in rescust service csar,
  95         # look in service-VcpesvcRescust1118-template.yml for groups vgw module metadata. TODO: read this value automatically
  96         # CHANGEME part
  97         self.vgw_VfModuleModelInvariantUuid = '26d6a718-17b2-4ba8-8691-c44343b2ecd2'
  98
  99         # OOM: this is the address that the brg and bng will nat for sdnc access - 10.0.0.x address of k8 host for sdnc-0 container
 100         self.sdnc_oam_ip = self.get_pod_node_oam_ip(self.sdnc_controller_pod)
 101         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
 102         self.oom_so_sdnc_aai_ip = self.get_pod_node_public_ip(self.sdnc_controller_pod)
 103         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
 104         self.oom_dcae_ves_collector = self.oom_so_sdnc_aai_ip
 105         # OOM: this is a k8s host external IP, e.g. oom-k8s-01 IP
 106         self.mr_ip_addr = self.oom_so_sdnc_aai_ip
 107         self.mr_ip_port = '30227'
 108         self.so_nbi_port = '30277' if self.oom_mode else '8080'
 109         self.sdnc_preloading_port = '30267' if self.oom_mode else '8282'
 110         self.aai_query_port = '30233' if self.oom_mode else '8443'
 111         self.sniro_port = '30288' if self.oom_mode else '8080'
 112
 113         self.host_names = ['sdc', 'so', 'sdnc', 'robot', 'aai-inst1', self.dcae_ves_collector_name]
 114         if extra_host_names:
 115             self.host_names.extend(extra_host_names)
 116         # get IP addresses
 117         self.hosts = self.get_vm_ip(self.host_names, self.external_net_addr, self.external_net_prefix_len)
 118         # this is the keyword used to name vgw stack, must not be used in other stacks
 119         self.vgw_name_keyword = 'base_vcpe_vgw'
 120         # this is the file that will keep the index of last assigned SO name
 121         self.vgw_vfmod_name_index_file= '__var/vgw_vfmod_name_index'
 122         self.svc_instance_uuid_file = '__var/svc_instance_uuid'
 123         self.preload_dict_file = '__var/preload_dict'
 124         self.vgmux_vnf_name_file = '__var/vgmux_vnf_name'
 125         self.product_family_id = 'f9457e8c-4afd-45da-9389-46acd9bf5116'
 126         self.custom_product_family_id = 'a9a77d5a-123e-4ca2-9eb9-0b015d2ee0fb'
 127         self.instance_name_prefix = {
 128             'service': 'vcpe_svc',
 129             'network': 'vcpe_net',
 130             'vnf': 'vcpe_vnf',
 131             'vfmodule': 'vcpe_vfmodule'
 132         }
 133         self.aai_userpass = 'AAI', 'AAI'
 134
 135         ############################################################################################################
 136         # following key is overriding public key from vCPE heat templates, it's important to use correct one in here
 137         # CHANGEME part
 138         self.pub_key = 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKXDgoo3+WOqcUG8/5uUbk81+yczgwC4Y8ywTmuQqbNxlY1oQ0YxdMUqUnhitSXs5S/yRuAVOYHwGg2mCs20oAINrP+mxBI544AMIb9itPjCtgqtE2EWo6MmnFGbHB4Sx3XioE7F4VPsh7japsIwzOjbrQe+Mua1TGQ5d4nfEOQaaglXLLPFfuc7WbhbJbK6Q7rHqZfRcOwAMXgDoBqlyqKeiKwnumddo2RyNT8ljYmvB6buz7KnMinzo7qB0uktVT05FH9Rg0CTWH5norlG5qXgP2aukL0gk1ph8iAt7uYLf1ktp+LJI2gaF6L0/qli9EmVCSLr1uJ38Q8CBflhkh'
 139
 140         self.os_tenant_id = self.cloud['--os-tenant-id']
 141         self.os_region_name = self.cloud['--os-region-name']
 142         self.common_preload_config['pub_key'] = self.pub_key
 143         self.sniro_url = 'http://' + self.hosts['robot'] + ':' + self.sniro_port + '/__admin/mappings'
 144         self.sniro_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 145         self.homing_solution = 'sniro'  # value is either 'sniro' or 'oof'
 146 #        self.homing_solution = 'oof'
 147         self.customer_location_used_by_oof = {
 148             "customerLatitude": "32.897480",
 149             "customerLongitude": "-97.040443",
 150             "customerName": "some_company"
 151         }
 152
 153         #############################################################################################
 154         # SDC urls
 155         self.sdc_be_port = '30204'
 156         self.sdc_be_request_userpass = 'vid', 'Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U'
 157         self.sdc_be_request_headers = {'X-ECOMP-InstanceID': 'VID'}
 158         self.sdc_be_url_prefix = 'https://' + self.hosts['sdc'] + ':' + self.sdc_be_port
 159         self.sdc_service_list_url = self.sdc_be_url_prefix + '/sdc/v1/catalog/services'
 160
 161         self.sdc_fe_port = '30207'
 162         self.sdc_fe_request_userpass = 'beep', 'boop'
 163         self.sdc_fe_request_headers = {'USER_ID': 'demo', 'Content-Type': 'application/json'}
 164         self.sdc_fe_url_prefix = 'https://' + self.hosts['sdc'] + ':' + self.sdc_fe_port
 165         self.sdc_get_category_list_url = self.sdc_fe_url_prefix + '/sdc1/feProxy/rest/v1/categories'
 166         self.sdc_create_allotted_resource_subcategory_url = self.sdc_fe_url_prefix + '/sdc1/feProxy/rest/v1/category/resources/resourceNewCategory.allotted%20resource/subCategory'
 167
 168         #############################################################################################
 169         # SDNC urls
 170         self.sdnc_userpass = 'admin', 'Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U'
 171         self.sdnc_db_name = 'sdnctl'
 172         self.sdnc_db_user = 'sdnctl'
 173         self.sdnc_db_pass = 'gamma'
 174         self.sdnc_db_port = '32774'
 175         self.sdnc_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 176         self.sdnc_preload_network_url = 'https://' + self.hosts['sdnc'] + \
 177                                         ':' + self.sdnc_preloading_port + '/restconf/operations/VNF-API:preload-network-topology-operation'
 178         self.sdnc_preload_vnf_url = 'https://' + self.hosts['sdnc'] + \
 179                                     ':' + self.sdnc_preloading_port + '/restconf/operations/VNF-API:preload-vnf-topology-operation'
 180         self.sdnc_preload_gra_url = 'https://' + self.hosts['sdnc'] + \
 181                                     ':' + self.sdnc_preloading_port + '/restconf/operations/GENERIC-RESOURCE-API:preload-vf-module-topology-operation'
 182         self.sdnc_ar_cleanup_url = 'https://' + self.hosts['sdnc'] + ':' + self.sdnc_preloading_port + \
 183                                    '/restconf/config/GENERIC-RESOURCE-API:'
 184
 185         #############################################################################################
 186         # SO urls, note: do NOT add a '/' at the end of the url
 187         self.so_req_api_url = {'v4': 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/serviceInstantiation/v7/serviceInstances',
 188                            'v5': 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/serviceInstantiation/v7/serviceInstances'}
 189         self.so_check_progress_api_url = 'http://' + self.hosts['so'] + ':' + self.so_nbi_port + '/onap/so/infra/orchestrationRequests/v6'
 190         self.so_userpass = 'InfraPortalClient', 'password1$'
 191         self.so_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 192         self.so_db_name = 'catalogdb'
 193         self.so_db_user = 'root'
 194         self.so_db_pass = 'secretpassword'
 195         self.so_db_port = '30252' if self.oom_mode else '32769'
 196
 197         self.vpp_inf_url = 'http://{0}:8183/restconf/config/ietf-interfaces:interfaces'
 198         self.vpp_api_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
 199         self.vpp_api_userpass = ('admin', 'admin')
 200         self.vpp_ves_url= 'http://{0}:8183/restconf/config/vesagent:vesagent'
 201
 202         #############################################################################################
 203         # POLICY urls
 204         self.policy_userpass = ('healthcheck', 'zb!XztG34')
 205         self.policy_headers = {'Accept': 'application/json', 'Content-Type': 'application/json'}
 206         self.policy_api_url = 'https://{0}:6969/policy/api/v1/policytypes/onap.policies.controlloop.Operational/versions/1.0.0/policies'
 207         self.policy_pap_get_url = 'https://{0}:6969/policy/pap/v1/pdps'
 208         self.policy_pap_json = {'policies': [{'policy-id': 'operational.vcpe'}]}
 209         self.policy_pap_post_url = self.policy_pap_get_url + '/policies'
 210         self.policy_api_service_name = 'policy-api'
 211         self.policy_pap_service_name = 'policy-pap'
 212
 213     def heatbridge(self, openstack_stack_name, svc_instance_uuid):
 214         """
 215         Add vserver information to AAI
 216         """
 217         self.logger.info('Adding vServer information to AAI for {0}'.format(openstack_stack_name))
 218         if not self.oom_mode:
 219             cmd = '/opt/demo.sh heatbridge {0} {1} vCPE'.format(openstack_stack_name, svc_instance_uuid)
 220             ret = commands.getstatusoutput("ssh -i onap_dev root@{0} '{1}'".format(self.hosts['robot'], cmd))
 221             self.logger.debug('%s', ret)
 222         else:
 223             print('To add vGMUX vserver info to AAI, do the following:')
 224             print('- ssh to rancher')
 225             print('- sudo su -')
 226             print('- cd /root/oom/kubernetes/robot')
 227             print('- ./demo-k8s.sh onap heatbridge {0} {1} vCPE'.format(openstack_stack_name, svc_instance_uuid))
 228
 229     def get_brg_mac_from_sdnc(self):
 230         """
 231         Check table DHCP_MAP in the SDNC DB. Find the newly instantiated BRG MAC address.
 232         Note that there might be multiple BRGs, the most recently instantiated BRG always has the largest IP address.
 233         """
 234         cnx = mysql.connector.connect(user=self.sdnc_db_user, password=self.sdnc_db_pass, database=self.sdnc_db_name,
 235                                       host=self.hosts['sdnc'], port=self.sdnc_db_port)
 236         cursor = cnx.cursor()
 237         query = "SELECT * from DHCP_MAP"
 238         cursor.execute(query)
 239
 240         self.logger.debug('DHCP_MAP table in SDNC')
 241         mac_recent = None
 242         host = -1
 243         for mac, ip in cursor:
 244             self.logger.debug(mac + ':' + ip)
 245             this_host = int(ip.split('.')[-1])
 246             if host < this_host:
 247                 host = this_host
 248                 mac_recent = mac
 249
 250         cnx.close()
 251
 252         assert mac_recent
 253         return mac_recent
 254
 255     def execute_cmds_sdnc_db(self, cmds):
 256         self.execute_cmds_db(cmds, self.sdnc_db_user, self.sdnc_db_pass, self.sdnc_db_name,
 257                              self.hosts['sdnc'], self.sdnc_db_port)
 258
 259     def execute_cmds_so_db(self, cmds):
 260         self.execute_cmds_db(cmds, self.so_db_user, self.so_db_pass, self.so_db_name,
 261                              self.hosts['so'], self.so_db_port)
 262
 263     def execute_cmds_db(self, cmds, dbuser, dbpass, dbname, host, port):
 264         cnx = mysql.connector.connect(user=dbuser, password=dbpass, database=dbname, host=host, port=port)
 265         cursor = cnx.cursor()
 266         for cmd in cmds:
 267             self.logger.debug(cmd)
 268             cursor.execute(cmd)
 269             self.logger.debug('%s', cursor)
 270         cnx.commit()
 271         cursor.close()
 272         cnx.close()
 273
 274     def find_file(self, file_name_keyword, file_ext, search_dir):
 275         """
 276         :param file_name_keyword:  keyword used to look for the csar file, case insensitive matching, e.g, infra
 277         :param file_ext: e.g., csar, json
 278         :param search_dir path to search
 279         :return: path name of the file
 280         """
 281         file_name_keyword = file_name_keyword.lower()
 282         file_ext = file_ext.lower()
 283         if not file_ext.startswith('.'):
 284             file_ext = '.' + file_ext
 285
 286         filenamepath = None
 287         for file_name in os.listdir(search_dir):
 288             file_name_lower = file_name.lower()
 289             if file_name_keyword in file_name_lower and file_name_lower.endswith(file_ext):
 290                 if filenamepath:
 291                     self.logger.error('Multiple files found for *{0}*.{1} in '
 292                                       'directory {2}'.format(file_name_keyword, file_ext, search_dir))
 293                     sys.exit()
 294                 filenamepath = os.path.abspath(os.path.join(search_dir, file_name))
 295
 296         if filenamepath:
 297             return filenamepath
 298         else:
 299             self.logger.error("Cannot find *{0}*{1} in directory {2}".format(file_name_keyword, file_ext, search_dir))
 300             sys.exit()
 301
 302     @staticmethod
 303     def network_name_to_subnet_name(network_name):
 304         """
 305         :param network_name: example: vcpe_net_cpe_signal_201711281221
 306         :return: vcpe_net_cpe_signal_subnet_201711281221
 307         """
 308         fields = network_name.split('_')
 309         fields.insert(-1, 'subnet')
 310         return '_'.join(fields)
 311
 312     def set_network_name(self, network_name):
 313         param = ' '.join([k + ' ' + v for k, v in self.cloud.items()])
 314         openstackcmd = 'openstack ' + param
 315         cmd = ' '.join([openstackcmd, 'network set --name', network_name, 'ONAP-NW1'])
 316         os.popen(cmd)
 317
 318     def set_subnet_name(self, network_name):
 319         """
 320         Example: network_name =  vcpe_net_cpe_signal_201711281221
 321         set subnet name to vcpe_net_cpe_signal_subnet_201711281221
 322         :return:
 323         """
 324         param = ' '.join([k + ' ' + v for k, v in self.cloud.items()])
 325         openstackcmd = 'openstack ' + param
 326
 327         # expected results: | subnets | subnet_id |
 328         subnet_info = os.popen(openstackcmd + ' network show ' + network_name + ' |grep subnets').read().split('|')
 329         if len(subnet_info) > 2 and subnet_info[1].strip() == 'subnets':
 330             subnet_id = subnet_info[2].strip()
 331             subnet_name = self.network_name_to_subnet_name(network_name)
 332             cmd = ' '.join([openstackcmd, 'subnet set --name', subnet_name, subnet_id])
 333             os.popen(cmd)
 334             self.logger.info("Subnet name set to: " + subnet_name)
 335             return True
 336         else:
 337             self.logger.error("Can't get subnet info from network name: " + network_name)
 338             return False
 339
 340     def set_closed_loop_policy(self, policy_template_file):
 341         # Gather policy services cluster ips
 342         p_api_cluster_ip = self.get_k8s_service_cluster_ip(self.policy_api_service_name)
 343         p_pap_cluster_ip = self.get_k8s_service_cluster_ip(self.policy_pap_service_name)
 344
 345         # Read policy json from file
 346         with open(policy_template_file) as f:
 347             try:
 348                 policy_json = json.load(f)
 349             except ValueError:
 350                 self.logger.error(policy_template_file + " doesn't seem to contain valid JSON data")
 351                 sys.exit()
 352
 353         # Check policy already applied
 354         requests.packages.urllib3.disable_warnings()
 355         policy_exists_req = requests.get(self.policy_pap_get_url.format(
 356                             p_pap_cluster_ip), auth=self.policy_userpass,
 357                             verify=False, headers=self.policy_headers)
 358         if policy_exists_req.status_code != 200:
 359             self.logger.error('Failure in checking CL policy existence. '
 360                                'Policy-pap responded with HTTP code {0}'.format(
 361                                policy_exists_req.status_code))
 362             sys.exit()
 363
 364         try:
 365             policy_exists_json = policy_exists_req.json()
 366         except ValueError as e:
 367             self.logger.error('Policy-pap request failed: ' + e.message)
 368             sys.exit()
 369
 370         try:
 371             assert policy_exists_json['groups'][0]['pdpSubgroups'] \
 372                                [1]['policies'][0]['name'] != 'operational.vcpe'
 373         except AssertionError:
 374             self.logger.info('vCPE closed loop policy already exists, not applying')
 375             return
 376         except IndexError:
 377             pass # policy doesn't exist
 378
 379         # Create policy
 380         policy_create_req = requests.post(self.policy_api_url.format(
 381                             p_api_cluster_ip), auth=self.policy_userpass,
 382                             json=policy_json, verify=False,
 383                             headers=self.policy_headers)
 384         # Get the policy id from policy-api response
 385         if policy_create_req.status_code != 200:
 386             self.logger.error('Failed creating policy. Policy-api responded'
 387                               ' with HTTP code {0}'.format(policy_create_req.status_code))
 388             sys.exit()
 389
 390         try:
 391             policy_version = json.loads(policy_create_req.text)['policy-version']
 392         except (KeyError, ValueError):
 393             self.logger.error('Policy API response not understood:')
 394             self.logger.debug('\n' + str(policy_create_req.text))
 395
 396         # Inject the policy into Policy PAP
 397         self.policy_pap_json['policies'].append({'policy-version': policy_version})
 398         policy_insert_req = requests.post(self.policy_pap_post_url.format(
 399                             p_pap_cluster_ip), auth=self.policy_userpass,
 400                             json=self.policy_pap_json, verify=False,
 401                             headers=self.policy_headers)
 402         if policy_insert_req.status_code != 200:
 403             self.logger.error('Policy PAP request failed with HTTP code'
 404                               '{0}'.format(policy_insert_req.status_code))
 405             sys.exit()
 406         self.logger.info('Successully pushed closed loop Policy')
 407
 408     def is_node_in_aai(self, node_type, node_uuid):
 409         key = None
 410         search_node_type = None
 411         if node_type == 'service':
 412             search_node_type = 'service-instance'
 413             key = 'service-instance-id'
 414         elif node_type == 'vnf':
 415             search_node_type = 'generic-vnf'
 416             key = 'vnf-id'
 417         else:
 418             logging.error('Invalid node_type: ' + node_type)
 419             sys.exit()
 420
 421         url = 'https://{0}:{1}/aai/v11/search/nodes-query?search-node-type={2}&filter={3}:EQUALS:{4}'.format(
 422             self.hosts['aai-inst1'], self.aai_query_port, search_node_type, key, node_uuid)
 423
 424         headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'X-FromAppID': 'vCPE-Robot', 'X-TransactionId': 'get_aai_subscr'}
 425         requests.packages.urllib3.disable_warnings()
 426         r = requests.get(url, headers=headers, auth=self.aai_userpass, verify=False)
 427         response = r.json()
 428         self.logger.debug('aai query: ' + url)
 429         self.logger.debug('aai response:\n' + json.dumps(response, indent=4, sort_keys=True))
 430         return 'result-data' in response
 431
 432     @staticmethod
 433     def extract_ip_from_str(net_addr, net_addr_len, sz):
 434         """
 435         :param net_addr:  e.g. 10.5.12.0
 436         :param net_addr_len: e.g. 24
 437         :param sz: a string
 438         :return: the first IP address matching the network, e.g. 10.5.12.3
 439         """
 440         network = ipaddress.ip_network(unicode('{0}/{1}'.format(net_addr, net_addr_len)), strict=False)
 441         ip_list = re.findall(r'[0-9]+(?:\.[0-9]+){3}', sz)
 442         for ip in ip_list:
 443             this_net = ipaddress.ip_network(unicode('{0}/{1}'.format(ip, net_addr_len)), strict=False)
 444             if this_net == network:
 445                 return str(ip)
 446         return None
 447
 448     def get_pod_node_oam_ip(self, pod):
 449         """
 450         :Assuming kubectl is available and configured by default config (~/.kube/config)
 451         :param pod: pod name substring, e.g. 'sdnc-sdnc-0'
 452         :return pod's cluster node oam ip (10.0.0.0/16)
 453         """
 454         ret = None
 455         config.load_kube_config()
 456         api = client.CoreV1Api()
 457         kslogger = logging.getLogger('kubernetes')
 458         kslogger.setLevel(logging.INFO)
 459         res = api.list_pod_for_all_namespaces()
 460         for i in res.items:
 461             if pod in i.metadata.name:
 462                 self.logger.debug("found {0}\t{1}\t{2}".format(i.metadata.name, i.status.host_ip, i.spec.node_name))
 463                 ret = i.status.host_ip
 464                 break
 465
 466         if ret is None:
 467             ret = raw_input("Enter " + self.sdnc_controller_pod + " pod cluster node OAM IP address(10.0.0.0/16): ")
 468         return ret
 469
 470     def get_pod_node_public_ip(self, pod):
 471         """
 472         :Assuming kubectl is available and configured by default config (~/.kube/config)
 473         :param pod: pod name substring, e.g. 'sdnc-sdnc-0'
 474         :return pod's cluster node public ip (i.e. 10.12.0.0/16)
 475         """
 476         ret = None
 477         config.load_kube_config()
 478         api = client.CoreV1Api()
 479         kslogger = logging.getLogger('kubernetes')
 480         kslogger.setLevel(logging.INFO)
 481         res = api.list_pod_for_all_namespaces()
 482         for i in res.items:
 483             if pod in i.metadata.name:
 484                 ret = self.get_vm_public_ip_by_nova(i.spec.node_name)
 485                 self.logger.debug("found node {0} public ip: {1}".format(i.spec.node_name, ret))
 486                 break
 487
 488         if ret is None:
 489             ret = raw_input("Enter " + self.sdnc_controller_pod + " pod cluster node public IP address(i.e. " + self.external_net_addr + "): ")
 490         return ret
 491
 492     def get_vm_public_ip_by_nova(self, vm):
 493         """
 494         This method uses openstack nova api to retrieve vm public ip
 495         :param vm: vm name
 496         :return vm public ip
 497         """
 498         subnet = IPNetwork('{0}/{1}'.format(self.external_net_addr, self.external_net_prefix_len))
 499         nova = openstackclient.Client(2, self.cloud['--os-username'], self.cloud['--os-password'], self.cloud['--os-tenant-id'], self.cloud['--os-auth-url'])
 500         for i in nova.servers.list():
 501             if i.name == vm:
 502                 for k, v in i.networks.items():
 503                     for ip in v:
 504                         if IPAddress(ip) in subnet:
 505                             return ip
 506         return None
 507
 508     def get_vm_ip(self, keywords, net_addr=None, net_addr_len=None):
 509         """
 510         :param keywords: list of keywords to search for vm, e.g. ['bng', 'gmux', 'brg']
 511         :param net_addr: e.g. 10.12.5.0
 512         :param net_addr_len: e.g. 24
 513         :return: dictionary {keyword: ip}
 514         """
 515         if not net_addr:
 516             net_addr = self.external_net_addr
 517
 518         if not net_addr_len:
 519             net_addr_len = self.external_net_prefix_len
 520
 521         param = ' '.join([k + ' ' + v for k, v in self.cloud.items() if 'identity' not in k])
 522         openstackcmd = 'nova ' + param + ' list'
 523         self.logger.debug(openstackcmd)
 524
 525         results = os.popen(openstackcmd).read()
 526         all_vm_ip_dict = self.extract_vm_ip_as_dict(results, net_addr, net_addr_len)
 527         latest_vm_list = self.remove_old_vms(all_vm_ip_dict.keys(), self.cpe_vm_prefix)
 528         latest_vm_ip_dict = {vm: all_vm_ip_dict[vm] for vm in latest_vm_list}
 529         ip_dict = self.select_subset_vm_ip(latest_vm_ip_dict, keywords)
 530         if self.oom_mode:
 531             ip_dict.update(self.get_oom_onap_vm_ip(keywords))
 532
 533         if len(ip_dict) != len(keywords):
 534             self.logger.error('Cannot find all desired IP addresses for %s.', keywords)
 535             self.logger.error(json.dumps(ip_dict, indent=4, sort_keys=True))
 536             self.logger.error('Temporarily continue.. remember to check back vcpecommon.py line: 396')
 537 #            sys.exit()
 538         return ip_dict
 539
 540     def get_oom_onap_vm_ip(self, keywords):
 541         vm_ip = {}
 542         onap_vm_list = set(['sdc', 'so', 'sdnc', 'aai-inst1', 'robot', self.dcae_ves_collector_name])
 543         for vm in keywords:
 544             if vm in onap_vm_list:
 545                 vm_ip[vm] = self.oom_so_sdnc_aai_ip
 546         return vm_ip
 547
 548     def get_k8s_service_cluster_ip(self, service):
 549         """
 550         Returns cluster IP for a given service
 551         :param service: name of the service
 552         :return: cluster ip
 553         """
 554         config.load_kube_config()
 555         api = client.CoreV1Api()
 556         kslogger = logging.getLogger('kubernetes')
 557         kslogger.setLevel(logging.INFO)
 558         try:
 559             resp = api.read_namespaced_service(service, self.onap_namespace)
 560         except client.rest.ApiException as e:
 561             self.logger.error('Error while making k8s API request: ' + e.body)
 562             sys.exit()
 563
 564         return resp.spec.cluster_ip
 565
 566     def extract_vm_ip_as_dict(self, novalist_results, net_addr, net_addr_len):
 567         vm_ip_dict = {}
 568         for line in novalist_results.split('\n'):
 569             fields = line.split('|')
 570             if len(fields) == 8:
 571                 vm_name = fields[2]
 572                 ip_info = fields[-2]
 573                 ip = self.extract_ip_from_str(net_addr, net_addr_len, ip_info)
 574                 vm_ip_dict[vm_name] = ip
 575
 576         return vm_ip_dict
 577
 578     def remove_old_vms(self, vm_list, prefix):
 579         """
 580         For vms with format name_timestamp, only keep the one with the latest timestamp.
 581         E.g.,
 582             zdcpe1cpe01brgemu01_201805222148        (drop this)
 583             zdcpe1cpe01brgemu01_201805222229        (keep this)
 584             zdcpe1cpe01gw01_201805162201
 585         """
 586         new_vm_list = []
 587         same_type_vm_dict = {}
 588         for vm in vm_list:
 589             fields = vm.split('_')
 590             if vm.startswith(prefix) and len(fields) == 2 and len(fields[-1]) == len('201805222148') and fields[-1].isdigit():
 591                 if vm > same_type_vm_dict.get(fields[0], '0'):
 592                     same_type_vm_dict[fields[0]] = vm
 593             else:
 594                 new_vm_list.append(vm)
 595
 596         new_vm_list.extend(same_type_vm_dict.values())
 597         return new_vm_list
 598
 599     def select_subset_vm_ip(self, all_vm_ip_dict, vm_name_keyword_list):
 600         vm_ip_dict = {}
 601         for keyword in vm_name_keyword_list:
 602             for vm, ip in all_vm_ip_dict.items():
 603                 if keyword in vm:
 604                     vm_ip_dict[keyword] = ip
 605                     break
 606         return vm_ip_dict
 607
 608     def del_vgmux_ves_mode(self):
 609         url = self.vpp_ves_url.format(self.hosts['mux']) + '/mode'
 610         r = requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 611         self.logger.debug('%s', r)
 612
 613     def del_vgmux_ves_collector(self):
 614         url = self.vpp_ves_url.format(self.hosts['mux']) + '/config'
 615         r = requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 616         self.logger.debug('%s', r)
 617
 618     def set_vgmux_ves_collector(self ):
 619         url = self.vpp_ves_url.format(self.hosts['mux'])
 620         data = {'config':
 621                     {'server-addr': self.hosts[self.dcae_ves_collector_name],
 622                      'server-port': '30235' if self.oom_mode else '8081',
 623                      'read-interval': '10',
 624                      'is-add':'1'
 625                      }
 626                 }
 627         r = requests.post(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass, json=data)
 628         self.logger.debug('%s', r)
 629
 630     def set_vgmux_packet_loss_rate(self, lossrate, vg_vnf_instance_name):
 631         url = self.vpp_ves_url.format(self.hosts['mux'])
 632         data = {"mode":
 633                     {"working-mode": "demo",
 634                      "base-packet-loss": str(lossrate),
 635                      "source-name": vg_vnf_instance_name
 636                      }
 637                 }
 638         r = requests.post(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass, json=data)
 639         self.logger.debug('%s', r)
 640
 641         # return all the VxLAN interface names of BRG or vGMUX based on the IP address
 642     def get_vxlan_interfaces(self, ip, print_info=False):
 643         url = self.vpp_inf_url.format(ip)
 644         self.logger.debug('url is this: %s', url)
 645         r = requests.get(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 646         data = r.json()['interfaces']['interface']
 647         if print_info:
 648             for inf in data:
 649                 if 'name' in inf and 'type' in inf and inf['type'] == 'v3po:vxlan-tunnel':
 650                     print(json.dumps(inf, indent=4, sort_keys=True))
 651
 652         return [inf['name'] for inf in data if 'name' in inf and 'type' in inf and inf['type'] == 'v3po:vxlan-tunnel']
 653
 654     # delete all VxLAN interfaces of each hosts
 655     def delete_vxlan_interfaces(self, host_dic):
 656         for host, ip in host_dic.items():
 657             deleted = False
 658             self.logger.info('{0}: Getting VxLAN interfaces'.format(host))
 659             inf_list = self.get_vxlan_interfaces(ip)
 660             for inf in inf_list:
 661                 deleted = True
 662                 time.sleep(2)
 663                 self.logger.info("{0}: Deleting VxLAN crossconnect {1}".format(host, inf))
 664                 url = self.vpp_inf_url.format(ip) + '/interface/' + inf + '/v3po:l2'
 665                 requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 666
 667             for inf in inf_list:
 668                 deleted = True
 669                 time.sleep(2)
 670                 self.logger.info("{0}: Deleting VxLAN interface {1}".format(host, inf))
 671                 url = self.vpp_inf_url.format(ip) + '/interface/' + inf
 672                 requests.delete(url, headers=self.vpp_api_headers, auth=self.vpp_api_userpass)
 673
 674             if len(self.get_vxlan_interfaces(ip)) > 0:
 675                 self.logger.error("Error deleting VxLAN from {0}, try to restart the VM, IP is {1}.".format(host, ip))
 676                 return False
 677
 678             if not deleted:
 679                 self.logger.info("{0}: no VxLAN interface found, nothing to delete".format(host))
 680         return True
 681
 682     @staticmethod
 683     def save_object(obj, filepathname):
 684         with open(filepathname, 'wb') as fout:
 685             pickle.dump(obj, fout)
 686
 687     @staticmethod
 688     def load_object(filepathname):
 689         with open(filepathname, 'rb') as fin:
 690             return pickle.load(fin)
 691
 692     @staticmethod
 693     def increase_ip_address_or_vni_in_template(vnf_template_file, vnf_parameter_name_list):
 694         with open(vnf_template_file) as json_input:
 695             json_data = json.load(json_input)
 696             param_list = json_data['VNF-API:input']['VNF-API:vnf-topology-information']['VNF-API:vnf-parameters']
 697             for param in param_list:
 698                 if param['vnf-parameter-name'] in vnf_parameter_name_list:
 699                     ipaddr_or_vni = param['vnf-parameter-value'].split('.')
 700                     number = int(ipaddr_or_vni[-1])
 701                     if 254 == number:
 702                         number = 10
 703                     else:
 704                         number = number + 1
 705                     ipaddr_or_vni[-1] = str(number)
 706                     param['vnf-parameter-value'] = '.'.join(ipaddr_or_vni)
 707
 708         assert json_data is not None
 709         with open(vnf_template_file, 'w') as json_output:
 710             json.dump(json_data, json_output, indent=4, sort_keys=True)
 711
 712     def save_preload_data(self, preload_data):
 713         self.save_object(preload_data, self.preload_dict_file)
 714
 715     def load_preload_data(self):
 716         return self.load_object(self.preload_dict_file)
 717
 718     def save_vgmux_vnf_name(self, vgmux_vnf_name):
 719         self.save_object(vgmux_vnf_name, self.vgmux_vnf_name_file)
 720
 721     def load_vgmux_vnf_name(self):
 722         return self.load_object(self.vgmux_vnf_name_file)
 723