catalog/packages/biz/vnf_package.py

   1 # Copyright 2018 ZTE Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import json
  16 import logging
  17 import os
  18 import sys
  19 import threading
  20 import traceback
  21 import urllib
  22 import uuid
  23 import zipfile
  24
  25 from catalog.packages import const
  26 from catalog.packages.biz.common import parse_file_range, read, save
  27 from catalog.packages.biz.notificationsutil import prepare_vnfpkg_notification, NotificationsUtil
  28 from catalog.pub.config.config import CATALOG_ROOT_PATH
  29 from catalog.pub.database.models import VnfPackageModel, NSPackageModel
  30 from catalog.pub.exceptions import CatalogException, ResourceNotFoundException
  31 from catalog.pub.utils import fileutil, toscaparser
  32 from catalog.pub.utils.values import ignore_case_get
  33
  34 logger = logging.getLogger(__name__)
  35
  36
  37 class VnfPackage(object):
  38
  39     def __init__(self):
  40         pass
  41
  42     def create_vnf_pkg(self, data):
  43         user_defined_data = ignore_case_get(data, "userDefinedData", {})
  44         vnf_pkg_id = str(uuid.uuid4())
  45         VnfPackageModel.objects.create(
  46             vnfPackageId=vnf_pkg_id,
  47             onboardingState=const.PKG_STATUS.CREATED,
  48             operationalState=const.PKG_STATUS.DISABLED,
  49             usageState=const.PKG_STATUS.NOT_IN_USE,
  50             userDefinedData=json.dumps(user_defined_data)
  51         )
  52         data = {
  53             "id": vnf_pkg_id,
  54             "onboardingState": const.PKG_STATUS.CREATED,
  55             "operationalState": const.PKG_STATUS.DISABLED,
  56             "usageState": const.PKG_STATUS.NOT_IN_USE,
  57             "userDefinedData": user_defined_data,
  58             "_links": None
  59         }
  60         return data
  61
  62     def query_multiple(self):
  63         pkgs_info = []
  64         nf_pkgs = VnfPackageModel.objects.filter()
  65         for nf_pkg in nf_pkgs:
  66             ret = fill_response_data(nf_pkg)
  67             pkgs_info.append(ret)
  68         return pkgs_info
  69
  70     def query_single(self, vnf_pkg_id):
  71         nf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
  72         if not nf_pkg.exists():
  73             logger.error('VNF package(%s) does not exist.' % vnf_pkg_id)
  74             raise ResourceNotFoundException('VNF package(%s) does not exist.' % vnf_pkg_id)
  75         return fill_response_data(nf_pkg[0])
  76
  77     def delete_vnf_pkg(self, vnf_pkg_id):
  78         vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
  79         if not vnf_pkg.exists():
  80             logger.debug('VNF package(%s) has been deleted.' % vnf_pkg_id)
  81             return
  82         '''
  83         if vnf_pkg[0].operationalState != PKG_STATUS.DISABLED:
  84             raise CatalogException("The VNF package (%s) is not disabled" % vnf_pkg_id)
  85         if vnf_pkg[0].usageState != PKG_STATUS.NOT_IN_USE:
  86             raise CatalogException("The VNF package (%s) is in use" % vnf_pkg_id)
  87         '''
  88         del_vnfd_id = vnf_pkg[0].vnfdId
  89         ns_pkgs = NSPackageModel.objects.all()
  90         for ns_pkg in ns_pkgs:
  91             nsd_model = None
  92             if ns_pkg.nsdModel:
  93                 nsd_model = json.JSONDecoder().decode(ns_pkg.nsdModel)
  94             if not nsd_model:
  95                 continue
  96             for vnf in nsd_model['vnfs']:
  97                 if del_vnfd_id == vnf["properties"]["descriptor_id"]:
  98                     raise CatalogException('VNFD(%s) is referenced.' % del_vnfd_id)
  99         vnf_pkg.delete()
 100         send_notification(vnf_pkg_id, const.PKG_NOTIFICATION_TYPE.CHANGE,
 101                           const.PKG_CHANGE_TYPE.PKG_DELETE)
 102
 103         vnf_pkg_path = os.path.join(CATALOG_ROOT_PATH, vnf_pkg_id)
 104         fileutil.delete_dirs(vnf_pkg_path)
 105         logger.info('VNF package(%s) has been deleted.' % vnf_pkg_id)
 106
 107     def upload(self, vnf_pkg_id, remote_file):
 108         logger.info('Start to upload VNF package(%s)...' % vnf_pkg_id)
 109         vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 110         # if vnf_pkg[0].onboardingState != PKG_STATUS.CREATED:
 111         #     logger.error("VNF package(%s) is not CREATED" % vnf_pkg_id)
 112         #     raise CatalogException("VNF package(%s) is not CREATED" % vnf_pkg_id)
 113         vnf_pkg.update(onboardingState=const.PKG_STATUS.UPLOADING)
 114         send_notification(vnf_pkg_id, const.PKG_NOTIFICATION_TYPE.ONBOARDING,
 115                           const.PKG_CHANGE_TYPE.OP_STATE_CHANGE)
 116
 117         local_file_name = save(remote_file, vnf_pkg_id)
 118         logger.info('VNF package(%s) has been uploaded.' % vnf_pkg_id)
 119         return local_file_name
 120
 121     def download(self, vnf_pkg_id, file_range):
 122         logger.info('Start to download VNF package(%s)...' % vnf_pkg_id)
 123         nf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 124         if not nf_pkg.exists():
 125             logger.error('VNF package(%s) does not exist.' % vnf_pkg_id)
 126             raise ResourceNotFoundException('VNF package(%s) does not exist.' % vnf_pkg_id)
 127         if nf_pkg[0].onboardingState != const.PKG_STATUS.ONBOARDED:
 128             raise CatalogException("VNF package (%s) is not on-boarded" % vnf_pkg_id)
 129
 130         local_file_path = nf_pkg[0].localFilePath
 131         start, end = parse_file_range(local_file_path, file_range)
 132         logger.info('VNF package (%s) has been downloaded.' % vnf_pkg_id)
 133         return read(local_file_path, start, end)
 134
 135     def download_vnfd(self, vnf_pkg_id):
 136         logger.info('Start to download VNFD of VNF package(%s)...' % vnf_pkg_id)
 137         nf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 138         if not nf_pkg.exists():
 139             logger.error('VNF package(%s) does not exist.' % vnf_pkg_id)
 140             raise ResourceNotFoundException('VNF package(%s) does not exist.' % vnf_pkg_id)
 141         if nf_pkg[0].onboardingState != const.PKG_STATUS.ONBOARDED:
 142             raise CatalogException("VNF package (%s) is not on-boarded" % vnf_pkg_id)
 143
 144         vnfd_zip_file = self.creat_vnfd(vnf_pkg_id, nf_pkg[0].localFilePath)
 145         logger.info('VNFD of VNF package (%s) has been downloaded.' % vnf_pkg_id)
 146         return read(vnfd_zip_file, 0, os.path.getsize(vnfd_zip_file))
 147
 148     def creat_vnfd(self, vnf_pkg_id, vendor_pkg_file):
 149         """
 150         Create VNFD zip file from vendor original package
 151         :param self:
 152         :param vnf_pkg_id: VNF package id (CSAR id)
 153         :param vendor_pkg_file: vendor original package
 154         :return:
 155         """
 156         vnf_package_path = os.path.join(CATALOG_ROOT_PATH, vnf_pkg_id)
 157         if not os.path.exists(vnf_package_path):
 158             os.makedirs(vnf_package_path)
 159         vnfd_zip_file = os.path.join(vnf_package_path, "VNFD.zip")
 160         if os.path.exists(vnfd_zip_file):
 161             return vnfd_zip_file
 162         else:
 163             if vendor_pkg_file.endswith(".csar") or vendor_pkg_file.endswith(".zip"):
 164                 try:
 165                     vnfd_path = os.path.join(vnf_package_path, "vnfd")
 166                     with zipfile.ZipFile(vendor_pkg_file, 'r') as vendor_zip:
 167                         vender_files = vendor_zip.namelist()
 168                         for vender_file in vender_files:
 169                             if str(vender_file).startswith("Definitions"):
 170                                 vendor_zip.extract(vender_file, vnfd_path)
 171                     with zipfile.ZipFile(vnfd_zip_file, 'w', zipfile.ZIP_DEFLATED) as vnfd_zip:
 172                         def_path = os.path.join(vnfd_path, "Definitions")
 173                         if os.path.exists(def_path):
 174                             def_files = os.listdir(def_path)
 175                             for def_file in def_files:
 176                                 full_path = os.path.join(def_path, def_file)
 177                                 vnfd_zip.write(full_path, def_file)
 178                     return vnfd_zip_file
 179                 except Exception as e:
 180                     logger.error(e)
 181                     if os.path.exists(vnfd_zip):
 182                         os.remove(vnfd_zip)
 183                     raise e
 184                 finally:
 185                     fileutil.delete_dirs(vnfd_path)
 186
 187
 188 class VnfPkgUploadThread(threading.Thread):
 189     def __init__(self, data, vnf_pkg_id):
 190         threading.Thread.__init__(self)
 191         self.vnf_pkg_id = vnf_pkg_id
 192         self.data = data
 193         self.upload_file_name = None
 194
 195     def run(self):
 196         try:
 197             self.upload_vnf_pkg_from_uri()
 198             parse_vnfd_and_save(self.vnf_pkg_id, self.upload_file_name)
 199         except CatalogException as e:
 200             logger.error(e.args[0])
 201         except Exception as e:
 202             logger.error(e.args[0])
 203             logger.error(traceback.format_exc())
 204             logger.error(str(sys.exc_info()))
 205
 206     def upload_vnf_pkg_from_uri(self):
 207         logger.info("Start to upload VNF packge(%s) from URI..." % self.vnf_pkg_id)
 208         vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=self.vnf_pkg_id)
 209         if vnf_pkg[0].onboardingState != const.PKG_STATUS.CREATED:
 210             logger.error("VNF package(%s) is not CREATED" % self.vnf_pkg_id)
 211             raise CatalogException("VNF package (%s) is not created" % self.vnf_pkg_id)
 212         vnf_pkg.update(onboardingState=const.PKG_STATUS.UPLOADING)
 213         send_notification(self.vnf_pkg_id, const.PKG_NOTIFICATION_TYPE.ONBOARDING,
 214                           const.PKG_CHANGE_TYPE.OP_STATE_CHANGE)
 215
 216         uri = ignore_case_get(self.data, "addressInformation")
 217         response = urllib.request.urlopen(uri)
 218
 219         local_file_dir = os.path.join(CATALOG_ROOT_PATH, self.vnf_pkg_id)
 220         self.upload_file_name = os.path.join(local_file_dir, os.path.basename(uri))
 221         if not os.path.exists(local_file_dir):
 222             fileutil.make_dirs(local_file_dir)
 223         with open(self.upload_file_name, "wt") as local_file:
 224             local_file.write(response.read())
 225         response.close()
 226         logger.info('VNF packge(%s) has been uploaded.' % self.vnf_pkg_id)
 227
 228
 229 def fill_response_data(nf_pkg):
 230     pkg_info = {}
 231     pkg_info["id"] = nf_pkg.vnfPackageId
 232     pkg_info["vnfdId"] = nf_pkg.vnfdId
 233     pkg_info["vnfProductName"] = nf_pkg.vnfdProductName
 234     pkg_info["vnfSoftwareVersion"] = nf_pkg.vnfSoftwareVersion
 235     pkg_info["vnfdVersion"] = nf_pkg.vnfdVersion
 236     if nf_pkg.checksum:
 237         pkg_info["checksum"] = json.JSONDecoder().decode(nf_pkg.checksum)
 238     pkg_info["softwareImages"] = None  # TODO
 239     pkg_info["additionalArtifacts"] = None  # TODO
 240     pkg_info["onboardingState"] = nf_pkg.onboardingState
 241     pkg_info["operationalState"] = nf_pkg.operationalState
 242     pkg_info["usageState"] = nf_pkg.usageState
 243     if nf_pkg.userDefinedData:
 244         pkg_info["userDefinedData"] = json.JSONDecoder().decode(nf_pkg.userDefinedData)
 245     pkg_info["_links"] = None  # TODO
 246     return pkg_info
 247
 248
 249 def parse_vnfd_and_save(vnf_pkg_id, vnf_pkg_path):
 250     logger.info('Start to process VNF package(%s)...' % vnf_pkg_id)
 251     vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 252     vnf_pkg.update(onboardingState=const.PKG_STATUS.PROCESSING)
 253     vnfd_json = toscaparser.parse_vnfd(vnf_pkg_path)
 254     vnfd = json.JSONDecoder().decode(vnfd_json)
 255
 256     if vnfd.get("vnf", "") != "":
 257         vnfd_id = vnfd["vnf"]["properties"].get("descriptor_id", "")
 258         other_pkg = VnfPackageModel.objects.filter(vnfdId=vnfd_id)
 259         if other_pkg and other_pkg[0].vnfPackageId != vnf_pkg_id:
 260             logger.error("VNF package(%s,%s) already exists.", other_pkg[0].vnfPackageId, vnfd_id)
 261             raise CatalogException("VNF package(%s) already exists." % vnfd_id)
 262         vnf_provider = vnfd["vnf"]["properties"].get("provider", "")
 263         vnfd_ver = vnfd["vnf"]["properties"].get("descriptor_version", "")
 264         vnf_software_version = vnfd["vnf"]["properties"].get("software_version", "")
 265         vnfd_product_name = vnfd["vnf"]["properties"].get("product_name", "")
 266         vnf_pkg.update(
 267             vnfPackageId=vnf_pkg_id,
 268             vnfdId=vnfd_id,
 269             vnfdProductName=vnfd_product_name,
 270             vnfVendor=vnf_provider,
 271             vnfdVersion=vnfd_ver,
 272             vnfSoftwareVersion=vnf_software_version,
 273             vnfdModel=vnfd_json,
 274             onboardingState=const.PKG_STATUS.ONBOARDED,
 275             operationalState=const.PKG_STATUS.ENABLED,
 276             usageState=const.PKG_STATUS.NOT_IN_USE,
 277             localFilePath=vnf_pkg_path,
 278             vnfPackageUri=os.path.split(vnf_pkg_path)[-1]
 279         )
 280     else:
 281         raise CatalogException("VNF propeties and metadata in VNF Package(id=%s) are empty." % vnf_pkg_id)
 282     logger.info('VNF package(%s) has been processed(done).' % vnf_pkg_id)
 283
 284
 285 def handle_upload_failed(vnf_pkg_id):
 286     vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 287     vnf_pkg.update(onboardingState=const.PKG_STATUS.CREATED)
 288
 289
 290 def send_notification(pkg_id, type, pkg_change_type, operational_state=None):
 291     data = prepare_vnfpkg_notification(vnf_pkg_id=pkg_id,
 292                                        notification_type=type,
 293                                        pkg_change_type=pkg_change_type,
 294                                        operational_state=operational_state)
 295     filters = {
 296         'vnfdId': 'vnfd_id',
 297         'vnfPkgId': 'vnf_pkg_id'
 298     }
 299     logger.debug('Notify request data = %s' % data)
 300     logger.debug('Notify request filters = %s' % filters)
 301     NotificationsUtil().send_notification(data, filters, True)