catalog/packages/biz/vnf_package.py

   1 # Copyright 2018 ZTE Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import json
  16 import logging
  17 import os
  18
  19 import threading
  20 import traceback
  21 import urllib
  22 import uuid
  23 import zipfile
  24
  25 from catalog.packages import const
  26 from catalog.packages.biz.common import parse_file_range, read, save
  27 from catalog.packages.biz.notificationsutil import PkgNotifications
  28 from catalog.pub.config.config import CATALOG_ROOT_PATH
  29 from catalog.pub.database.models import VnfPackageModel, NSPackageModel
  30 from catalog.pub.exceptions import CatalogException, ResourceNotFoundException
  31 from catalog.pub.utils import fileutil, toscaparser
  32 from catalog.pub.utils.values import ignore_case_get
  33
  34 logger = logging.getLogger(__name__)
  35
  36
  37 class VnfPackage(object):
  38
  39     def __init__(self):
  40         pass
  41
  42     def create_vnf_pkg(self, data):
  43         user_defined_data = ignore_case_get(data, "userDefinedData", {})
  44         vnf_pkg_id = str(uuid.uuid4())
  45         VnfPackageModel.objects.create(
  46             vnfPackageId=vnf_pkg_id,
  47             onboardingState=const.PKG_STATUS.CREATED,
  48             operationalState=const.PKG_STATUS.DISABLED,
  49             usageState=const.PKG_STATUS.NOT_IN_USE,
  50             userDefinedData=json.dumps(user_defined_data)
  51         )
  52         data = {
  53             "id": vnf_pkg_id,
  54             "onboardingState": const.PKG_STATUS.CREATED,
  55             "operationalState": const.PKG_STATUS.DISABLED,
  56             "usageState": const.PKG_STATUS.NOT_IN_USE,
  57             "userDefinedData": user_defined_data,
  58             "_links": None
  59         }
  60         return data
  61
  62     def query_multiple(self):
  63         pkgs_info = []
  64         nf_pkgs = VnfPackageModel.objects.filter()
  65         for nf_pkg in nf_pkgs:
  66             ret = fill_response_data(nf_pkg)
  67             pkgs_info.append(ret)
  68         return pkgs_info
  69
  70     def query_single(self, vnf_pkg_id):
  71         nf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
  72         if not nf_pkg.exists():
  73             logger.error('VNF package(%s) does not exist.' % vnf_pkg_id)
  74             raise ResourceNotFoundException('VNF package(%s) does not exist.' % vnf_pkg_id)
  75         return fill_response_data(nf_pkg[0])
  76
  77     def delete_vnf_pkg(self, vnf_pkg_id):
  78         vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
  79         if not vnf_pkg.exists():
  80             logger.debug('VNF package(%s) has been deleted.' % vnf_pkg_id)
  81             return
  82         '''
  83         if vnf_pkg[0].operationalState != PKG_STATUS.DISABLED:
  84             raise CatalogException("The VNF package (%s) is not disabled" % vnf_pkg_id)
  85         if vnf_pkg[0].usageState != PKG_STATUS.NOT_IN_USE:
  86             raise CatalogException("The VNF package (%s) is in use" % vnf_pkg_id)
  87         '''
  88         del_vnfd_id = vnf_pkg[0].vnfdId
  89         ns_pkgs = NSPackageModel.objects.all()
  90         for ns_pkg in ns_pkgs:
  91             nsd_model = None
  92             if ns_pkg.nsdModel:
  93                 nsd_model = json.JSONDecoder().decode(ns_pkg.nsdModel)
  94             if not nsd_model:
  95                 continue
  96             for vnf in nsd_model['vnfs']:
  97                 if del_vnfd_id == vnf["properties"]["descriptor_id"]:
  98                     raise CatalogException('VNFD(%s) is referenced.' % del_vnfd_id)
  99         vnf_pkg.delete()
 100         send_notification(vnf_pkg_id, const.PKG_NOTIFICATION_TYPE.CHANGE,
 101                           const.PKG_CHANGE_TYPE.PKG_DELETE)
 102
 103         vnf_pkg_path = os.path.join(CATALOG_ROOT_PATH, vnf_pkg_id)
 104         fileutil.delete_dirs(vnf_pkg_path)
 105         logger.info('VNF package(%s) has been deleted.' % vnf_pkg_id)
 106
 107     def upload(self, vnf_pkg_id, remote_file):
 108         logger.info('Start to upload VNF package(%s)...' % vnf_pkg_id)
 109         vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 110         # if vnf_pkg[0].onboardingState != PKG_STATUS.CREATED:
 111         #     logger.error("VNF package(%s) is not CREATED" % vnf_pkg_id)
 112         #     raise CatalogException("VNF package(%s) is not CREATED" % vnf_pkg_id)
 113         vnf_pkg.update(onboardingState=const.PKG_STATUS.UPLOADING)
 114
 115         local_file_name = save(remote_file, vnf_pkg_id)
 116         logger.info('VNF package(%s) has been uploaded.' % vnf_pkg_id)
 117         return local_file_name
 118
 119     def download(self, vnf_pkg_id, file_range):
 120         logger.info('Start to download VNF package(%s)...' % vnf_pkg_id)
 121         nf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 122         if not nf_pkg.exists():
 123             logger.error('VNF package(%s) does not exist.' % vnf_pkg_id)
 124             raise ResourceNotFoundException('VNF package(%s) does not exist.' % vnf_pkg_id)
 125         if nf_pkg[0].onboardingState != const.PKG_STATUS.ONBOARDED:
 126             raise CatalogException("VNF package (%s) is not on-boarded" % vnf_pkg_id)
 127
 128         local_file_path = nf_pkg[0].localFilePath
 129         start, end = parse_file_range(local_file_path, file_range)
 130         logger.info('VNF package (%s) has been downloaded.' % vnf_pkg_id)
 131         return read(local_file_path, start, end)
 132
 133     def download_vnfd(self, vnf_pkg_id):
 134         logger.info('Start to download VNFD of VNF package(%s)...' % vnf_pkg_id)
 135         nf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 136         if not nf_pkg.exists():
 137             logger.error('VNF package(%s) does not exist.' % vnf_pkg_id)
 138             raise ResourceNotFoundException('VNF package(%s) does not exist.' % vnf_pkg_id)
 139         if nf_pkg[0].onboardingState != const.PKG_STATUS.ONBOARDED:
 140             raise CatalogException("VNF package (%s) is not on-boarded" % vnf_pkg_id)
 141
 142         vnfd_zip_file = self.creat_vnfd(vnf_pkg_id, nf_pkg[0].localFilePath)
 143         logger.info('VNFD of VNF package (%s) has been downloaded.' % vnf_pkg_id)
 144         return read(vnfd_zip_file, 0, os.path.getsize(vnfd_zip_file))
 145
 146     def creat_vnfd(self, vnf_pkg_id, vendor_pkg_file):
 147         """
 148         Create VNFD zip file from vendor original package
 149         :param self:
 150         :param vnf_pkg_id: VNF package id (CSAR id)
 151         :param vendor_pkg_file: vendor original package
 152         :return:
 153         """
 154         vnf_package_path = os.path.join(CATALOG_ROOT_PATH, vnf_pkg_id)
 155         if not os.path.exists(vnf_package_path):
 156             os.makedirs(vnf_package_path)
 157         vnfd_zip_file = os.path.join(vnf_package_path, "VNFD.zip")
 158         if os.path.exists(vnfd_zip_file):
 159             return vnfd_zip_file
 160         else:
 161             if vendor_pkg_file.endswith(".csar") or vendor_pkg_file.endswith(".zip"):
 162                 try:
 163                     vnfd_path = os.path.join(vnf_package_path, "vnfd")
 164                     with zipfile.ZipFile(vendor_pkg_file, 'r') as vendor_zip:
 165                         vender_files = vendor_zip.namelist()
 166                         for vender_file in vender_files:
 167                             if str(vender_file).startswith("Definitions"):
 168                                 vendor_zip.extract(vender_file, vnfd_path)
 169                     with zipfile.ZipFile(vnfd_zip_file, 'w', zipfile.ZIP_DEFLATED) as vnfd_zip:
 170                         def_path = os.path.join(vnfd_path, "Definitions")
 171                         if os.path.exists(def_path):
 172                             def_files = os.listdir(def_path)
 173                             for def_file in def_files:
 174                                 full_path = os.path.join(def_path, def_file)
 175                                 vnfd_zip.write(full_path, def_file)
 176                     return vnfd_zip_file
 177                 except Exception as e:
 178                     logger.error(e)
 179                     if os.path.exists(vnfd_zip):
 180                         os.remove(vnfd_zip)
 181                     raise e
 182                 finally:
 183                     fileutil.delete_dirs(vnfd_path)
 184
 185
 186 class VnfPkgUploadThread(threading.Thread):
 187     def __init__(self, data, vnf_pkg_id):
 188         threading.Thread.__init__(self)
 189         self.vnf_pkg_id = vnf_pkg_id
 190         self.data = data
 191         self.upload_file_name = None
 192
 193     def vnf_pkg_upload_failed_handle(self, error_msg):
 194         logger.error(error_msg)
 195         logger.error(traceback.format_exc())
 196         vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=self.vnf_pkg_id)
 197         if vnf_pkg and vnf_pkg[0].onboardingState == const.PKG_STATUS.UPLOADING:
 198             vnf_pkg.update(onboardingState=const.PKG_STATUS.CREATED)
 199
 200     def run(self):
 201         try:
 202             self.upload_vnf_pkg_from_uri()
 203             parse_vnfd_and_save(self.vnf_pkg_id, self.upload_file_name)
 204         except CatalogException as e:
 205             self.vnf_pkg_upload_failed_handle(e.args[0])
 206         except Exception as e:
 207             self.vnf_pkg_upload_failed_handle(e.args[0])
 208
 209     def upload_vnf_pkg_from_uri(self):
 210         logger.info("Start to upload VNF packge(%s) from URI..." % self.vnf_pkg_id)
 211         vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=self.vnf_pkg_id)
 212         if vnf_pkg[0].onboardingState != const.PKG_STATUS.CREATED:
 213             logger.error("VNF package(%s) is not CREATED" % self.vnf_pkg_id)
 214             raise CatalogException("VNF package (%s) is not created" % self.vnf_pkg_id)
 215         vnf_pkg.update(onboardingState=const.PKG_STATUS.UPLOADING)
 216         send_notification(self.vnf_pkg_id, const.PKG_NOTIFICATION_TYPE.ONBOARDING,
 217                           const.PKG_CHANGE_TYPE.OP_STATE_CHANGE)
 218
 219         uri = ignore_case_get(self.data, "addressInformation")
 220         response = urllib.request.urlopen(uri)
 221
 222         local_file_dir = os.path.join(CATALOG_ROOT_PATH, self.vnf_pkg_id)
 223         self.upload_file_name = os.path.join(local_file_dir, os.path.basename(uri))
 224         if not os.path.exists(local_file_dir):
 225             fileutil.make_dirs(local_file_dir)
 226         with open(self.upload_file_name, "wt") as local_file:
 227             local_file.write(response.read())
 228         response.close()
 229         logger.info('VNF packge(%s) has been uploaded.' % self.vnf_pkg_id)
 230
 231
 232 def get_mfile_data(path):
 233     logger.debug('get_mfile_data path %s' % path)
 234     files = fileutil.filter_files(path, '.mf')
 235     if files:
 236         src_file = os.path.join(path, files[0])
 237         src_dict_list = []
 238         with open(src_file, 'r') as f:
 239             data = f.readlines()
 240             for line in data:
 241                 if line.strip() == "":
 242                     continue
 243                 src_dict = {}
 244                 k, v = line.split(':', maxsplit=1)
 245                 if k.strip() in ["Source", "Algorithm", "Hash"]:
 246                     if k.strip() == "Source" and src_dict:
 247                         src_dict_list.extend(src_dict)
 248                         src_dict = {}
 249                     src_dict[k.strip()] = v.strip()
 250                     print("src_dict:%s" % src_dict)
 251         if src_dict:
 252             src_dict_list.append(src_dict)
 253
 254         logger.debug('get_mfile_data: %s' % src_dict_list)
 255         return src_dict_list
 256
 257
 258 def fill_artifacts_data(vnf_pkg_id):
 259     vnf_pkg_path = os.path.join(CATALOG_ROOT_PATH, vnf_pkg_id)
 260     if os.path.exists(vnf_pkg_path) is False:
 261         return None
 262     files = fileutil.filter_files(vnf_pkg_path, '.csar')
 263     for filename in files:
 264         logger.info('fill_artifacts_data filename (%s)...' % filename)
 265         dst_file_path = os.path.join(vnf_pkg_path, "tmp")
 266         src_file = os.path.join(vnf_pkg_path, filename)
 267         dst_file = os.path.join(dst_file_path, filename)
 268         fileutil.recreate_dir(dst_file_path)
 269         fileutil.copy(src_file, vnf_pkg_path, dst_file)
 270         artifact_vnf_file = fileutil.unzip_file(dst_file, dst_file_path, "")
 271         artifacts = get_mfile_data(artifact_vnf_file)
 272         if artifacts:
 273             return [{
 274                 "artifactPath": artifact.get("Source", ""),
 275                 "checksum": {
 276                     "algorithm": artifact.get("Hash", "Null"),
 277                     "hash": artifact.get("Algorithm", "Null")
 278                 }
 279             } for artifact in artifacts]
 280
 281
 282 def fill_response_data(nf_pkg):
 283     pkg_info = {}
 284     pkg_info["id"] = nf_pkg.vnfPackageId
 285     pkg_info["vnfdId"] = nf_pkg.vnfdId
 286     pkg_info["vnfProductName"] = nf_pkg.vnfdProductName
 287     pkg_info["vnfSoftwareVersion"] = nf_pkg.vnfSoftwareVersion
 288     pkg_info["vnfdVersion"] = nf_pkg.vnfdVersion
 289     if nf_pkg.checksum:
 290         pkg_info["checksum"] = json.JSONDecoder().decode(nf_pkg.checksum)
 291     pkg_info["softwareImages"] = None  # TODO
 292     pkg_info["additionalArtifacts"] = fill_artifacts_data(nf_pkg.vnfPackageId)
 293     pkg_info["onboardingState"] = nf_pkg.onboardingState
 294     pkg_info["operationalState"] = nf_pkg.operationalState
 295     pkg_info["usageState"] = nf_pkg.usageState
 296     if nf_pkg.userDefinedData:
 297         pkg_info["userDefinedData"] = json.JSONDecoder().decode(nf_pkg.userDefinedData)
 298     pkg_info["_links"] = None  # TODO
 299     return pkg_info
 300
 301
 302 def parse_vnfd_and_save(vnf_pkg_id, vnf_pkg_path):
 303     logger.info('Start to process VNF package(%s)...' % vnf_pkg_id)
 304     vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 305     vnf_pkg.update(onboardingState=const.PKG_STATUS.PROCESSING)
 306     vnfd_json = toscaparser.parse_vnfd(vnf_pkg_path)
 307     vnfd = json.JSONDecoder().decode(vnfd_json)
 308
 309     if vnfd.get("vnf", "") != "":
 310         vnfd_id = vnfd["vnf"]["properties"].get("descriptor_id", "")
 311         other_pkg = VnfPackageModel.objects.filter(vnfdId=vnfd_id)
 312         if other_pkg and other_pkg[0].vnfPackageId != vnf_pkg_id:
 313             logger.error("VNF package(%s,%s) already exists.", other_pkg[0].vnfPackageId, vnfd_id)
 314             raise CatalogException("VNF package(%s) already exists." % vnfd_id)
 315         vnf_provider = vnfd["vnf"]["properties"].get("provider", "")
 316         vnfd_ver = vnfd["vnf"]["properties"].get("descriptor_version", "")
 317         vnf_software_version = vnfd["vnf"]["properties"].get("software_version", "")
 318         vnfd_product_name = vnfd["vnf"]["properties"].get("product_name", "")
 319         vnf_pkg.update(
 320             vnfPackageId=vnf_pkg_id,
 321             vnfdId=vnfd_id,
 322             vnfdProductName=vnfd_product_name,
 323             vnfVendor=vnf_provider,
 324             vnfdVersion=vnfd_ver,
 325             vnfSoftwareVersion=vnf_software_version,
 326             vnfdModel=vnfd_json,
 327             onboardingState=const.PKG_STATUS.ONBOARDED,
 328             operationalState=const.PKG_STATUS.ENABLED,
 329             usageState=const.PKG_STATUS.NOT_IN_USE,
 330             localFilePath=vnf_pkg_path,
 331             vnfPackageUri=os.path.split(vnf_pkg_path)[-1]
 332         )
 333         send_notification(vnf_pkg_id, const.PKG_NOTIFICATION_TYPE.ONBOARDING,
 334                           const.PKG_CHANGE_TYPE.OP_STATE_CHANGE)
 335     else:
 336         raise CatalogException("VNF propeties and metadata in VNF Package(id=%s) are empty." % vnf_pkg_id)
 337     logger.info('VNF package(%s) has been processed(done).' % vnf_pkg_id)
 338
 339
 340 def handle_upload_failed(vnf_pkg_id):
 341     vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 342     vnf_pkg.update(onboardingState=const.PKG_STATUS.CREATED)
 343
 344
 345 def send_notification(pkg_id, type, pkg_change_type, operational_state=None):
 346     notify = PkgNotifications(type, pkg_id, change_type=pkg_change_type,
 347                               operational_state=operational_state)
 348     notify.send_notification()