catalog/packages/biz/vnf_package.py

   1 # Copyright 2018 ZTE Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import json
  16 import logging
  17 import os
  18
  19 import threading
  20 import traceback
  21 import urllib
  22 import uuid
  23 import zipfile
  24
  25 from catalog.packages import const
  26 from catalog.packages.biz.common import parse_file_range, read, save
  27 from catalog.packages.biz.notificationsutil import prepare_vnfpkg_notification, NotificationsUtil
  28 from catalog.pub.config.config import CATALOG_ROOT_PATH
  29 from catalog.pub.database.models import VnfPackageModel, NSPackageModel
  30 from catalog.pub.exceptions import CatalogException, ResourceNotFoundException
  31 from catalog.pub.utils import fileutil, toscaparser
  32 from catalog.pub.utils.values import ignore_case_get
  33
  34 logger = logging.getLogger(__name__)
  35
  36
  37 class VnfPackage(object):
  38
  39     def __init__(self):
  40         pass
  41
  42     def create_vnf_pkg(self, data):
  43         user_defined_data = ignore_case_get(data, "userDefinedData", {})
  44         vnf_pkg_id = str(uuid.uuid4())
  45         VnfPackageModel.objects.create(
  46             vnfPackageId=vnf_pkg_id,
  47             onboardingState=const.PKG_STATUS.CREATED,
  48             operationalState=const.PKG_STATUS.DISABLED,
  49             usageState=const.PKG_STATUS.NOT_IN_USE,
  50             userDefinedData=json.dumps(user_defined_data)
  51         )
  52         data = {
  53             "id": vnf_pkg_id,
  54             "onboardingState": const.PKG_STATUS.CREATED,
  55             "operationalState": const.PKG_STATUS.DISABLED,
  56             "usageState": const.PKG_STATUS.NOT_IN_USE,
  57             "userDefinedData": user_defined_data,
  58             "_links": None
  59         }
  60         return data
  61
  62     def query_multiple(self):
  63         pkgs_info = []
  64         nf_pkgs = VnfPackageModel.objects.filter()
  65         for nf_pkg in nf_pkgs:
  66             ret = fill_response_data(nf_pkg)
  67             pkgs_info.append(ret)
  68         return pkgs_info
  69
  70     def query_single(self, vnf_pkg_id):
  71         nf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
  72         if not nf_pkg.exists():
  73             logger.error('VNF package(%s) does not exist.' % vnf_pkg_id)
  74             raise ResourceNotFoundException('VNF package(%s) does not exist.' % vnf_pkg_id)
  75         return fill_response_data(nf_pkg[0])
  76
  77     def delete_vnf_pkg(self, vnf_pkg_id):
  78         vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
  79         if not vnf_pkg.exists():
  80             logger.debug('VNF package(%s) has been deleted.' % vnf_pkg_id)
  81             return
  82         '''
  83         if vnf_pkg[0].operationalState != PKG_STATUS.DISABLED:
  84             raise CatalogException("The VNF package (%s) is not disabled" % vnf_pkg_id)
  85         if vnf_pkg[0].usageState != PKG_STATUS.NOT_IN_USE:
  86             raise CatalogException("The VNF package (%s) is in use" % vnf_pkg_id)
  87         '''
  88         del_vnfd_id = vnf_pkg[0].vnfdId
  89         ns_pkgs = NSPackageModel.objects.all()
  90         for ns_pkg in ns_pkgs:
  91             nsd_model = None
  92             if ns_pkg.nsdModel:
  93                 nsd_model = json.JSONDecoder().decode(ns_pkg.nsdModel)
  94             if not nsd_model:
  95                 continue
  96             for vnf in nsd_model['vnfs']:
  97                 if del_vnfd_id == vnf["properties"]["descriptor_id"]:
  98                     raise CatalogException('VNFD(%s) is referenced.' % del_vnfd_id)
  99         vnf_pkg.delete()
 100         send_notification(vnf_pkg_id, const.PKG_NOTIFICATION_TYPE.CHANGE,
 101                           const.PKG_CHANGE_TYPE.PKG_DELETE)
 102
 103         vnf_pkg_path = os.path.join(CATALOG_ROOT_PATH, vnf_pkg_id)
 104         fileutil.delete_dirs(vnf_pkg_path)
 105         logger.info('VNF package(%s) has been deleted.' % vnf_pkg_id)
 106
 107     def upload(self, vnf_pkg_id, remote_file):
 108         logger.info('Start to upload VNF package(%s)...' % vnf_pkg_id)
 109         vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 110         # if vnf_pkg[0].onboardingState != PKG_STATUS.CREATED:
 111         #     logger.error("VNF package(%s) is not CREATED" % vnf_pkg_id)
 112         #     raise CatalogException("VNF package(%s) is not CREATED" % vnf_pkg_id)
 113         vnf_pkg.update(onboardingState=const.PKG_STATUS.UPLOADING)
 114         send_notification(vnf_pkg_id, const.PKG_NOTIFICATION_TYPE.ONBOARDING,
 115                           const.PKG_CHANGE_TYPE.OP_STATE_CHANGE)
 116
 117         local_file_name = save(remote_file, vnf_pkg_id)
 118         logger.info('VNF package(%s) has been uploaded.' % vnf_pkg_id)
 119         return local_file_name
 120
 121     def download(self, vnf_pkg_id, file_range):
 122         logger.info('Start to download VNF package(%s)...' % vnf_pkg_id)
 123         nf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 124         if not nf_pkg.exists():
 125             logger.error('VNF package(%s) does not exist.' % vnf_pkg_id)
 126             raise ResourceNotFoundException('VNF package(%s) does not exist.' % vnf_pkg_id)
 127         if nf_pkg[0].onboardingState != const.PKG_STATUS.ONBOARDED:
 128             raise CatalogException("VNF package (%s) is not on-boarded" % vnf_pkg_id)
 129
 130         local_file_path = nf_pkg[0].localFilePath
 131         start, end = parse_file_range(local_file_path, file_range)
 132         logger.info('VNF package (%s) has been downloaded.' % vnf_pkg_id)
 133         return read(local_file_path, start, end)
 134
 135     def download_vnfd(self, vnf_pkg_id):
 136         logger.info('Start to download VNFD of VNF package(%s)...' % vnf_pkg_id)
 137         nf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 138         if not nf_pkg.exists():
 139             logger.error('VNF package(%s) does not exist.' % vnf_pkg_id)
 140             raise ResourceNotFoundException('VNF package(%s) does not exist.' % vnf_pkg_id)
 141         if nf_pkg[0].onboardingState != const.PKG_STATUS.ONBOARDED:
 142             raise CatalogException("VNF package (%s) is not on-boarded" % vnf_pkg_id)
 143
 144         vnfd_zip_file = self.creat_vnfd(vnf_pkg_id, nf_pkg[0].localFilePath)
 145         logger.info('VNFD of VNF package (%s) has been downloaded.' % vnf_pkg_id)
 146         return read(vnfd_zip_file, 0, os.path.getsize(vnfd_zip_file))
 147
 148     def creat_vnfd(self, vnf_pkg_id, vendor_pkg_file):
 149         """
 150         Create VNFD zip file from vendor original package
 151         :param self:
 152         :param vnf_pkg_id: VNF package id (CSAR id)
 153         :param vendor_pkg_file: vendor original package
 154         :return:
 155         """
 156         vnf_package_path = os.path.join(CATALOG_ROOT_PATH, vnf_pkg_id)
 157         if not os.path.exists(vnf_package_path):
 158             os.makedirs(vnf_package_path)
 159         vnfd_zip_file = os.path.join(vnf_package_path, "VNFD.zip")
 160         if os.path.exists(vnfd_zip_file):
 161             return vnfd_zip_file
 162         else:
 163             if vendor_pkg_file.endswith(".csar") or vendor_pkg_file.endswith(".zip"):
 164                 try:
 165                     vnfd_path = os.path.join(vnf_package_path, "vnfd")
 166                     with zipfile.ZipFile(vendor_pkg_file, 'r') as vendor_zip:
 167                         vender_files = vendor_zip.namelist()
 168                         for vender_file in vender_files:
 169                             if str(vender_file).startswith("Definitions"):
 170                                 vendor_zip.extract(vender_file, vnfd_path)
 171                     with zipfile.ZipFile(vnfd_zip_file, 'w', zipfile.ZIP_DEFLATED) as vnfd_zip:
 172                         def_path = os.path.join(vnfd_path, "Definitions")
 173                         if os.path.exists(def_path):
 174                             def_files = os.listdir(def_path)
 175                             for def_file in def_files:
 176                                 full_path = os.path.join(def_path, def_file)
 177                                 vnfd_zip.write(full_path, def_file)
 178                     return vnfd_zip_file
 179                 except Exception as e:
 180                     logger.error(e)
 181                     if os.path.exists(vnfd_zip):
 182                         os.remove(vnfd_zip)
 183                     raise e
 184                 finally:
 185                     fileutil.delete_dirs(vnfd_path)
 186
 187
 188 class VnfPkgUploadThread(threading.Thread):
 189     def __init__(self, data, vnf_pkg_id):
 190         threading.Thread.__init__(self)
 191         self.vnf_pkg_id = vnf_pkg_id
 192         self.data = data
 193         self.upload_file_name = None
 194
 195     def vnf_pkg_upload_failed_handle(self, error_msg):
 196         logger.error(error_msg)
 197         logger.error(traceback.format_exc())
 198         vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=self.vnf_pkg_id)
 199         if vnf_pkg and vnf_pkg[0].onboardingState == const.PKG_STATUS.UPLOADING:
 200             vnf_pkg.update(onboardingState=const.PKG_STATUS.CREATED)
 201
 202     def run(self):
 203         try:
 204             self.upload_vnf_pkg_from_uri()
 205             parse_vnfd_and_save(self.vnf_pkg_id, self.upload_file_name)
 206         except CatalogException as e:
 207             self.vnf_pkg_upload_failed_handle(e.args[0])
 208         except Exception as e:
 209             self.vnf_pkg_upload_failed_handle(e.args[0])
 210
 211     def upload_vnf_pkg_from_uri(self):
 212         logger.info("Start to upload VNF packge(%s) from URI..." % self.vnf_pkg_id)
 213         vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=self.vnf_pkg_id)
 214         if vnf_pkg[0].onboardingState != const.PKG_STATUS.CREATED:
 215             logger.error("VNF package(%s) is not CREATED" % self.vnf_pkg_id)
 216             raise CatalogException("VNF package (%s) is not created" % self.vnf_pkg_id)
 217         vnf_pkg.update(onboardingState=const.PKG_STATUS.UPLOADING)
 218         send_notification(self.vnf_pkg_id, const.PKG_NOTIFICATION_TYPE.ONBOARDING,
 219                           const.PKG_CHANGE_TYPE.OP_STATE_CHANGE)
 220
 221         uri = ignore_case_get(self.data, "addressInformation")
 222         response = urllib.request.urlopen(uri)
 223
 224         local_file_dir = os.path.join(CATALOG_ROOT_PATH, self.vnf_pkg_id)
 225         self.upload_file_name = os.path.join(local_file_dir, os.path.basename(uri))
 226         if not os.path.exists(local_file_dir):
 227             fileutil.make_dirs(local_file_dir)
 228         with open(self.upload_file_name, "wt") as local_file:
 229             local_file.write(response.read())
 230         response.close()
 231         logger.info('VNF packge(%s) has been uploaded.' % self.vnf_pkg_id)
 232
 233
 234 def fill_response_data(nf_pkg):
 235     pkg_info = {}
 236     pkg_info["id"] = nf_pkg.vnfPackageId
 237     pkg_info["vnfdId"] = nf_pkg.vnfdId
 238     pkg_info["vnfProductName"] = nf_pkg.vnfdProductName
 239     pkg_info["vnfSoftwareVersion"] = nf_pkg.vnfSoftwareVersion
 240     pkg_info["vnfdVersion"] = nf_pkg.vnfdVersion
 241     if nf_pkg.checksum:
 242         pkg_info["checksum"] = json.JSONDecoder().decode(nf_pkg.checksum)
 243     pkg_info["softwareImages"] = None  # TODO
 244     pkg_info["additionalArtifacts"] = None  # TODO
 245     pkg_info["onboardingState"] = nf_pkg.onboardingState
 246     pkg_info["operationalState"] = nf_pkg.operationalState
 247     pkg_info["usageState"] = nf_pkg.usageState
 248     if nf_pkg.userDefinedData:
 249         pkg_info["userDefinedData"] = json.JSONDecoder().decode(nf_pkg.userDefinedData)
 250     pkg_info["_links"] = None  # TODO
 251     return pkg_info
 252
 253
 254 def parse_vnfd_and_save(vnf_pkg_id, vnf_pkg_path):
 255     logger.info('Start to process VNF package(%s)...' % vnf_pkg_id)
 256     vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 257     vnf_pkg.update(onboardingState=const.PKG_STATUS.PROCESSING)
 258     vnfd_json = toscaparser.parse_vnfd(vnf_pkg_path)
 259     vnfd = json.JSONDecoder().decode(vnfd_json)
 260
 261     if vnfd.get("vnf", "") != "":
 262         vnfd_id = vnfd["vnf"]["properties"].get("descriptor_id", "")
 263         other_pkg = VnfPackageModel.objects.filter(vnfdId=vnfd_id)
 264         if other_pkg and other_pkg[0].vnfPackageId != vnf_pkg_id:
 265             logger.error("VNF package(%s,%s) already exists.", other_pkg[0].vnfPackageId, vnfd_id)
 266             raise CatalogException("VNF package(%s) already exists." % vnfd_id)
 267         vnf_provider = vnfd["vnf"]["properties"].get("provider", "")
 268         vnfd_ver = vnfd["vnf"]["properties"].get("descriptor_version", "")
 269         vnf_software_version = vnfd["vnf"]["properties"].get("software_version", "")
 270         vnfd_product_name = vnfd["vnf"]["properties"].get("product_name", "")
 271         vnf_pkg.update(
 272             vnfPackageId=vnf_pkg_id,
 273             vnfdId=vnfd_id,
 274             vnfdProductName=vnfd_product_name,
 275             vnfVendor=vnf_provider,
 276             vnfdVersion=vnfd_ver,
 277             vnfSoftwareVersion=vnf_software_version,
 278             vnfdModel=vnfd_json,
 279             onboardingState=const.PKG_STATUS.ONBOARDED,
 280             operationalState=const.PKG_STATUS.ENABLED,
 281             usageState=const.PKG_STATUS.NOT_IN_USE,
 282             localFilePath=vnf_pkg_path,
 283             vnfPackageUri=os.path.split(vnf_pkg_path)[-1]
 284         )
 285     else:
 286         raise CatalogException("VNF propeties and metadata in VNF Package(id=%s) are empty." % vnf_pkg_id)
 287     logger.info('VNF package(%s) has been processed(done).' % vnf_pkg_id)
 288
 289
 290 def handle_upload_failed(vnf_pkg_id):
 291     vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 292     vnf_pkg.update(onboardingState=const.PKG_STATUS.CREATED)
 293
 294
 295 def send_notification(pkg_id, type, pkg_change_type, operational_state=None):
 296     data = prepare_vnfpkg_notification(vnf_pkg_id=pkg_id,
 297                                        notification_type=type,
 298                                        pkg_change_type=pkg_change_type,
 299                                        operational_state=operational_state)
 300     filters = {
 301         'vnfdId': 'vnfd_id',
 302         'vnfPkgId': 'vnf_pkg_id'
 303     }
 304     logger.debug('Notify request data = %s' % data)
 305     logger.debug('Notify request filters = %s' % filters)
 306     NotificationsUtil().send_notification(data, filters, True)