catalog/packages/biz/vnf_package.py

   1 # Copyright 2018 ZTE Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import json
  16 import logging
  17 import os
  18
  19 import threading
  20 import traceback
  21 import urllib
  22 import uuid
  23 import zipfile
  24
  25 from catalog.packages import const
  26 from catalog.packages.biz.common import parse_file_range, read, save
  27 from catalog.packages.biz.notificationsutil import PkgNotifications
  28 from catalog.pub.config.config import CATALOG_ROOT_PATH
  29 from catalog.pub.database.models import VnfPackageModel, NSPackageModel
  30 from catalog.pub.exceptions import CatalogException, ResourceNotFoundException
  31 from catalog.pub.utils import fileutil, toscaparser
  32 from catalog.pub.utils.values import ignore_case_get
  33
  34 logger = logging.getLogger(__name__)
  35
  36
  37 class VnfPackage(object):
  38
  39     def __init__(self):
  40         pass
  41
  42     def create_vnf_pkg(self, data):
  43         user_defined_data = ignore_case_get(data, "userDefinedData", {})
  44         vnf_pkg_id = str(uuid.uuid4())
  45         VnfPackageModel.objects.create(
  46             vnfPackageId=vnf_pkg_id,
  47             onboardingState=const.PKG_STATUS.CREATED,
  48             operationalState=const.PKG_STATUS.DISABLED,
  49             usageState=const.PKG_STATUS.NOT_IN_USE,
  50             userDefinedData=json.dumps(user_defined_data)
  51         )
  52         data = {
  53             "id": vnf_pkg_id,
  54             "onboardingState": const.PKG_STATUS.CREATED,
  55             "operationalState": const.PKG_STATUS.DISABLED,
  56             "usageState": const.PKG_STATUS.NOT_IN_USE,
  57             "userDefinedData": user_defined_data,
  58             "_links": None
  59         }
  60         return data
  61
  62     def query_multiple(self):
  63         pkgs_info = []
  64         nf_pkgs = VnfPackageModel.objects.filter()
  65         for nf_pkg in nf_pkgs:
  66             ret = fill_response_data(nf_pkg)
  67             pkgs_info.append(ret)
  68         return pkgs_info
  69
  70     def query_single(self, vnf_pkg_id):
  71         nf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
  72         if not nf_pkg.exists():
  73             logger.error('VNF package(%s) does not exist.' % vnf_pkg_id)
  74             raise ResourceNotFoundException('VNF package(%s) does not exist.' % vnf_pkg_id)
  75         return fill_response_data(nf_pkg[0])
  76
  77     def delete_vnf_pkg(self, vnf_pkg_id):
  78         vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
  79         if not vnf_pkg.exists():
  80             logger.debug('VNF package(%s) has been deleted.' % vnf_pkg_id)
  81             return
  82         '''
  83         if vnf_pkg[0].operationalState != PKG_STATUS.DISABLED:
  84             raise CatalogException("The VNF package (%s) is not disabled" % vnf_pkg_id)
  85         if vnf_pkg[0].usageState != PKG_STATUS.NOT_IN_USE:
  86             raise CatalogException("The VNF package (%s) is in use" % vnf_pkg_id)
  87         '''
  88         del_vnfd_id = vnf_pkg[0].vnfdId
  89         ns_pkgs = NSPackageModel.objects.all()
  90         for ns_pkg in ns_pkgs:
  91             nsd_model = None
  92             if ns_pkg.nsdModel:
  93                 nsd_model = json.JSONDecoder().decode(ns_pkg.nsdModel)
  94             if not nsd_model:
  95                 continue
  96             for vnf in nsd_model['vnfs']:
  97                 if del_vnfd_id == vnf["properties"]["descriptor_id"]:
  98                     raise CatalogException('VNFD(%s) is referenced.' % del_vnfd_id)
  99         vnf_pkg.delete()
 100         send_notification(vnf_pkg_id, const.PKG_NOTIFICATION_TYPE.CHANGE,
 101                           const.PKG_CHANGE_TYPE.PKG_DELETE)
 102
 103         vnf_pkg_path = os.path.join(CATALOG_ROOT_PATH, vnf_pkg_id)
 104         fileutil.delete_dirs(vnf_pkg_path)
 105         logger.info('VNF package(%s) has been deleted.' % vnf_pkg_id)
 106
 107     def upload(self, vnf_pkg_id, remote_file):
 108         logger.info('Start to upload VNF package(%s)...' % vnf_pkg_id)
 109         vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 110         # if vnf_pkg[0].onboardingState != PKG_STATUS.CREATED:
 111         #     logger.error("VNF package(%s) is not CREATED" % vnf_pkg_id)
 112         #     raise CatalogException("VNF package(%s) is not CREATED" % vnf_pkg_id)
 113         vnf_pkg.update(onboardingState=const.PKG_STATUS.UPLOADING)
 114
 115         local_file_name = save(remote_file, vnf_pkg_id)
 116         logger.info('VNF package(%s) has been uploaded.' % vnf_pkg_id)
 117         return local_file_name
 118
 119     def download(self, vnf_pkg_id, file_range):
 120         logger.info('Start to download VNF package(%s)...' % vnf_pkg_id)
 121         nf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 122         if not nf_pkg.exists():
 123             logger.error('VNF package(%s) does not exist.' % vnf_pkg_id)
 124             raise ResourceNotFoundException('VNF package(%s) does not exist.' % vnf_pkg_id)
 125         if nf_pkg[0].onboardingState != const.PKG_STATUS.ONBOARDED:
 126             raise CatalogException("VNF package (%s) is not on-boarded" % vnf_pkg_id)
 127
 128         local_file_path = nf_pkg[0].localFilePath
 129         start, end = parse_file_range(local_file_path, file_range)
 130         logger.info('VNF package (%s) has been downloaded.' % vnf_pkg_id)
 131         return read(local_file_path, start, end)
 132
 133     def download_vnfd(self, vnf_pkg_id):
 134         logger.info('Start to download VNFD of VNF package(%s)...' % vnf_pkg_id)
 135         nf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 136         if not nf_pkg.exists():
 137             logger.error('VNF package(%s) does not exist.' % vnf_pkg_id)
 138             raise ResourceNotFoundException('VNF package(%s) does not exist.' % vnf_pkg_id)
 139         if nf_pkg[0].onboardingState != const.PKG_STATUS.ONBOARDED:
 140             raise CatalogException("VNF package (%s) is not on-boarded" % vnf_pkg_id)
 141
 142         vnfd_zip_file = self.creat_vnfd(vnf_pkg_id, nf_pkg[0].localFilePath)
 143         logger.info('VNFD of VNF package (%s) has been downloaded.' % vnf_pkg_id)
 144         return read(vnfd_zip_file, 0, os.path.getsize(vnfd_zip_file))
 145
 146     def creat_vnfd(self, vnf_pkg_id, vendor_pkg_file):
 147         """
 148         Create VNFD zip file from vendor original package
 149         :param self:
 150         :param vnf_pkg_id: VNF package id (CSAR id)
 151         :param vendor_pkg_file: vendor original package
 152         :return:
 153         """
 154         vnf_package_path = os.path.join(CATALOG_ROOT_PATH, vnf_pkg_id)
 155         if not os.path.exists(vnf_package_path):
 156             os.makedirs(vnf_package_path)
 157         vnfd_zip_file = os.path.join(vnf_package_path, "VNFD.zip")
 158         if os.path.exists(vnfd_zip_file):
 159             return vnfd_zip_file
 160         else:
 161             if vendor_pkg_file.endswith(".csar") or vendor_pkg_file.endswith(".zip"):
 162                 try:
 163                     vnfd_path = os.path.join(vnf_package_path, "vnfd")
 164                     with zipfile.ZipFile(vendor_pkg_file, 'r') as vendor_zip:
 165                         vender_files = vendor_zip.namelist()
 166                         for vender_file in vender_files:
 167                             if str(vender_file).startswith("Definitions"):
 168                                 vendor_zip.extract(vender_file, vnfd_path)
 169                     with zipfile.ZipFile(vnfd_zip_file, 'w', zipfile.ZIP_DEFLATED) as vnfd_zip:
 170                         def_path = os.path.join(vnfd_path, "Definitions")
 171                         if os.path.exists(def_path):
 172                             def_files = os.listdir(def_path)
 173                             for def_file in def_files:
 174                                 full_path = os.path.join(def_path, def_file)
 175                                 vnfd_zip.write(full_path, def_file)
 176                     return vnfd_zip_file
 177                 except Exception as e:
 178                     logger.error(e)
 179                     if os.path.exists(vnfd_zip):
 180                         os.remove(vnfd_zip)
 181                     raise e
 182                 finally:
 183                     fileutil.delete_dirs(vnfd_path)
 184
 185
 186 class VnfPkgUploadThread(threading.Thread):
 187     def __init__(self, data, vnf_pkg_id):
 188         threading.Thread.__init__(self)
 189         self.vnf_pkg_id = vnf_pkg_id
 190         self.data = data
 191         self.upload_file_name = None
 192
 193     def vnf_pkg_upload_failed_handle(self, error_msg):
 194         logger.error(error_msg)
 195         logger.error(traceback.format_exc())
 196         vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=self.vnf_pkg_id)
 197         if vnf_pkg and vnf_pkg[0].onboardingState == const.PKG_STATUS.UPLOADING:
 198             vnf_pkg.update(onboardingState=const.PKG_STATUS.CREATED)
 199
 200     def run(self):
 201         try:
 202             self.upload_vnf_pkg_from_uri()
 203             parse_vnfd_and_save(self.vnf_pkg_id, self.upload_file_name)
 204         except CatalogException as e:
 205             self.vnf_pkg_upload_failed_handle(e.args[0])
 206         except Exception as e:
 207             self.vnf_pkg_upload_failed_handle(e.args[0])
 208
 209     def upload_vnf_pkg_from_uri(self):
 210         logger.info("Start to upload VNF packge(%s) from URI..." % self.vnf_pkg_id)
 211         vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=self.vnf_pkg_id)
 212         if vnf_pkg[0].onboardingState != const.PKG_STATUS.CREATED:
 213             logger.error("VNF package(%s) is not CREATED" % self.vnf_pkg_id)
 214             raise CatalogException("VNF package (%s) is not created" % self.vnf_pkg_id)
 215         vnf_pkg.update(onboardingState=const.PKG_STATUS.UPLOADING)
 216         send_notification(self.vnf_pkg_id, const.PKG_NOTIFICATION_TYPE.ONBOARDING,
 217                           const.PKG_CHANGE_TYPE.OP_STATE_CHANGE)
 218
 219         uri = ignore_case_get(self.data, "addressInformation")
 220         response = urllib.request.urlopen(uri)
 221
 222         local_file_dir = os.path.join(CATALOG_ROOT_PATH, self.vnf_pkg_id)
 223         self.upload_file_name = os.path.join(local_file_dir, os.path.basename(uri))
 224         if not os.path.exists(local_file_dir):
 225             fileutil.make_dirs(local_file_dir)
 226         with open(self.upload_file_name, "wt") as local_file:
 227             local_file.write(response.read())
 228         response.close()
 229         logger.info('VNF packge(%s) has been uploaded.' % self.vnf_pkg_id)
 230
 231
 232 def fill_response_data(nf_pkg):
 233     pkg_info = {}
 234     pkg_info["id"] = nf_pkg.vnfPackageId
 235     pkg_info["vnfdId"] = nf_pkg.vnfdId
 236     pkg_info["vnfProductName"] = nf_pkg.vnfdProductName
 237     pkg_info["vnfSoftwareVersion"] = nf_pkg.vnfSoftwareVersion
 238     pkg_info["vnfdVersion"] = nf_pkg.vnfdVersion
 239     if nf_pkg.checksum:
 240         pkg_info["checksum"] = json.JSONDecoder().decode(nf_pkg.checksum)
 241     pkg_info["softwareImages"] = None  # TODO
 242     pkg_info["additionalArtifacts"] = None  # TODO
 243     pkg_info["onboardingState"] = nf_pkg.onboardingState
 244     pkg_info["operationalState"] = nf_pkg.operationalState
 245     pkg_info["usageState"] = nf_pkg.usageState
 246     if nf_pkg.userDefinedData:
 247         pkg_info["userDefinedData"] = json.JSONDecoder().decode(nf_pkg.userDefinedData)
 248     pkg_info["_links"] = None  # TODO
 249     return pkg_info
 250
 251
 252 def parse_vnfd_and_save(vnf_pkg_id, vnf_pkg_path):
 253     logger.info('Start to process VNF package(%s)...' % vnf_pkg_id)
 254     vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 255     vnf_pkg.update(onboardingState=const.PKG_STATUS.PROCESSING)
 256     vnfd_json = toscaparser.parse_vnfd(vnf_pkg_path)
 257     vnfd = json.JSONDecoder().decode(vnfd_json)
 258
 259     if vnfd.get("vnf", "") != "":
 260         vnfd_id = vnfd["vnf"]["properties"].get("descriptor_id", "")
 261         other_pkg = VnfPackageModel.objects.filter(vnfdId=vnfd_id)
 262         if other_pkg and other_pkg[0].vnfPackageId != vnf_pkg_id:
 263             logger.error("VNF package(%s,%s) already exists.", other_pkg[0].vnfPackageId, vnfd_id)
 264             raise CatalogException("VNF package(%s) already exists." % vnfd_id)
 265         vnf_provider = vnfd["vnf"]["properties"].get("provider", "")
 266         vnfd_ver = vnfd["vnf"]["properties"].get("descriptor_version", "")
 267         vnf_software_version = vnfd["vnf"]["properties"].get("software_version", "")
 268         vnfd_product_name = vnfd["vnf"]["properties"].get("product_name", "")
 269         vnf_pkg.update(
 270             vnfPackageId=vnf_pkg_id,
 271             vnfdId=vnfd_id,
 272             vnfdProductName=vnfd_product_name,
 273             vnfVendor=vnf_provider,
 274             vnfdVersion=vnfd_ver,
 275             vnfSoftwareVersion=vnf_software_version,
 276             vnfdModel=vnfd_json,
 277             onboardingState=const.PKG_STATUS.ONBOARDED,
 278             operationalState=const.PKG_STATUS.ENABLED,
 279             usageState=const.PKG_STATUS.NOT_IN_USE,
 280             localFilePath=vnf_pkg_path,
 281             vnfPackageUri=os.path.split(vnf_pkg_path)[-1]
 282         )
 283         send_notification(vnf_pkg_id, const.PKG_NOTIFICATION_TYPE.ONBOARDING,
 284                           const.PKG_CHANGE_TYPE.OP_STATE_CHANGE)
 285     else:
 286         raise CatalogException("VNF propeties and metadata in VNF Package(id=%s) are empty." % vnf_pkg_id)
 287     logger.info('VNF package(%s) has been processed(done).' % vnf_pkg_id)
 288
 289
 290 def handle_upload_failed(vnf_pkg_id):
 291     vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 292     vnf_pkg.update(onboardingState=const.PKG_STATUS.CREATED)
 293
 294
 295 def send_notification(pkg_id, type, pkg_change_type, operational_state=None):
 296     notify = PkgNotifications(type, pkg_id, change_type=pkg_change_type,
 297                               operational_state=operational_state)
 298     notify.send_notification()