catalog/packages/biz/vnf_package.py

   1 # Copyright 2018 ZTE Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import json
  16 import logging
  17 import os
  18 import threading
  19 import traceback
  20 import urllib
  21 import uuid
  22 import zipfile
  23
  24 from catalog.packages import const
  25 from catalog.packages.biz.common import parse_file_range, read, save
  26 from catalog.packages.biz.notificationsutil import PkgNotifications
  27 from catalog.pub.config.config import CATALOG_ROOT_PATH
  28 from catalog.pub.database.models import VnfPackageModel, NSPackageModel
  29 from catalog.pub.exceptions import CatalogException, ResourceNotFoundException
  30 from catalog.pub.utils import fileutil, toscaparser
  31 from catalog.pub.utils.values import ignore_case_get
  32
  33 logger = logging.getLogger(__name__)
  34
  35
  36 class VnfPackage(object):
  37     """
  38     The class for VNF package management
  39     """
  40
  41     def __init__(self):
  42         pass
  43
  44     def create_vnf_pkg(self, data):
  45         """
  46         Create a VNF package
  47         :param data: user defined data
  48         :return: VNF package info
  49         """
  50         user_defined_data = ignore_case_get(data, "userDefinedData", {})
  51         vnf_pkg_id = str(uuid.uuid4())
  52         VnfPackageModel.objects.create(
  53             vnfPackageId=vnf_pkg_id,
  54             onboardingState=const.PKG_STATUS.CREATED,
  55             operationalState=const.PKG_STATUS.DISABLED,
  56             usageState=const.PKG_STATUS.NOT_IN_USE,
  57             userDefinedData=json.dumps(user_defined_data)
  58         )
  59         data = {
  60             "id": vnf_pkg_id,
  61             "onboardingState": const.PKG_STATUS.CREATED,
  62             "operationalState": const.PKG_STATUS.DISABLED,
  63             "usageState": const.PKG_STATUS.NOT_IN_USE,
  64             "userDefinedData": user_defined_data,
  65             "_links": None
  66         }
  67         return data
  68
  69     def query_multiple(self):
  70         """
  71         Query the list of VNF package
  72         :return: The list of VNF pakcage
  73         """
  74         pkgs_info = []
  75         nf_pkgs = VnfPackageModel.objects.filter()
  76         for nf_pkg in nf_pkgs:
  77             ret = fill_response_data(nf_pkg)
  78             pkgs_info.append(ret)
  79         return pkgs_info
  80
  81     def query_single(self, vnf_pkg_id):
  82         """
  83         Query a single VNF package by given id
  84         :param vnf_pkg_id: The id of VNF package
  85         :return: VNF pckage info
  86         """
  87         nf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
  88         if not nf_pkg.exists():
  89             logger.error('VNF package(%s) does not exist.' % vnf_pkg_id)
  90             raise ResourceNotFoundException('VNF package(%s) does not exist.' % vnf_pkg_id)
  91         return fill_response_data(nf_pkg[0])
  92
  93     def delete_vnf_pkg(self, vnf_pkg_id):
  94         """
  95         Delete a VNF package by give id
  96         :param vnf_pkg_id: The id of VNF package
  97         :return:
  98         """
  99         vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 100         if not vnf_pkg.exists():
 101             logger.debug('VNF package(%s) has been deleted.' % vnf_pkg_id)
 102             return
 103         '''
 104         if vnf_pkg[0].operationalState != PKG_STATUS.DISABLED:
 105             raise CatalogException("The VNF package (%s) is not disabled" % vnf_pkg_id)
 106         if vnf_pkg[0].usageState != PKG_STATUS.NOT_IN_USE:
 107             raise CatalogException("The VNF package (%s) is in use" % vnf_pkg_id)
 108         '''
 109         del_vnfd_id = vnf_pkg[0].vnfdId
 110         ns_pkgs = NSPackageModel.objects.all()
 111         for ns_pkg in ns_pkgs:
 112             nsd_model = None
 113             if ns_pkg.nsdModel:
 114                 nsd_model = json.JSONDecoder().decode(ns_pkg.nsdModel)
 115             if not nsd_model:
 116                 continue
 117             for vnf in nsd_model['vnfs']:
 118                 if del_vnfd_id == vnf["properties"]["descriptor_id"]:
 119                     raise CatalogException('VNFD(%s) is referenced.' % del_vnfd_id)
 120         vnf_pkg.delete()
 121         send_notification(vnf_pkg_id, const.PKG_NOTIFICATION_TYPE.CHANGE,
 122                           const.PKG_CHANGE_TYPE.PKG_DELETE)
 123
 124         vnf_pkg_path = os.path.join(CATALOG_ROOT_PATH, vnf_pkg_id)
 125         fileutil.delete_dirs(vnf_pkg_path)
 126         logger.info('VNF package(%s) has been deleted.' % vnf_pkg_id)
 127
 128     def upload(self, vnf_pkg_id, remote_file):
 129         """
 130         Update VNF pckage file for given id
 131         :param vnf_pkg_id: The id of VNF package
 132         :param remote_file: VNF package file
 133         :return:
 134         """
 135         logger.info('Start to upload VNF package(%s)...' % vnf_pkg_id)
 136         vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 137         # if vnf_pkg[0].onboardingState != PKG_STATUS.CREATED:
 138         #     logger.error("VNF package(%s) is not CREATED" % vnf_pkg_id)
 139         #     raise CatalogException("VNF package(%s) is not CREATED" % vnf_pkg_id)
 140         vnf_pkg.update(onboardingState=const.PKG_STATUS.UPLOADING)
 141
 142         local_file_name = save(remote_file, vnf_pkg_id)
 143         logger.info('VNF package(%s) has been uploaded.' % vnf_pkg_id)
 144         return local_file_name
 145
 146     def download(self, vnf_pkg_id, file_range):
 147         logger.info('Start to download VNF package(%s)...' % vnf_pkg_id)
 148         nf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 149         if not nf_pkg.exists():
 150             logger.error('VNF package(%s) does not exist.' % vnf_pkg_id)
 151             raise ResourceNotFoundException('VNF package(%s) does not exist.' % vnf_pkg_id)
 152         if nf_pkg[0].onboardingState != const.PKG_STATUS.ONBOARDED:
 153             raise CatalogException("VNF package (%s) is not on-boarded" % vnf_pkg_id)
 154
 155         local_file_path = nf_pkg[0].localFilePath
 156         start, end = parse_file_range(local_file_path, file_range)
 157         logger.info('VNF package (%s) has been downloaded.' % vnf_pkg_id)
 158         return read(local_file_path, start, end)
 159
 160     def download_vnfd(self, vnf_pkg_id):
 161         """
 162         Download VNFD for given id
 163         :param vnf_pkg_id: The id of VNF package
 164         :return: VNFD
 165         """
 166         logger.info('Start to download VNFD of VNF package(%s)...' % vnf_pkg_id)
 167         nf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 168         if not nf_pkg.exists():
 169             logger.error('VNF package(%s) does not exist.' % vnf_pkg_id)
 170             raise ResourceNotFoundException('VNF package(%s) does not exist.' % vnf_pkg_id)
 171         if nf_pkg[0].onboardingState != const.PKG_STATUS.ONBOARDED:
 172             raise CatalogException("VNF package (%s) is not on-boarded" % vnf_pkg_id)
 173
 174         vnfd_zip_file = self.creat_vnfd(vnf_pkg_id, nf_pkg[0].localFilePath)
 175         logger.info('VNFD of VNF package (%s) has been downloaded.' % vnf_pkg_id)
 176         return read(vnfd_zip_file, 0, os.path.getsize(vnfd_zip_file))
 177
 178     def creat_vnfd(self, vnf_pkg_id, vendor_pkg_file):
 179         """
 180         Create VNFD zip file from vendor original package
 181         :param self:
 182         :param vnf_pkg_id: VNF package id (CSAR id)
 183         :param vendor_pkg_file: vendor original package
 184         :return:
 185         """
 186         vnf_package_path = os.path.join(CATALOG_ROOT_PATH, vnf_pkg_id)
 187         if not os.path.exists(vnf_package_path):
 188             os.makedirs(vnf_package_path)
 189         vnfd_zip_file = os.path.join(vnf_package_path, "VNFD.zip")
 190         if os.path.exists(vnfd_zip_file):
 191             return vnfd_zip_file
 192         else:
 193             if vendor_pkg_file.endswith(".csar") or vendor_pkg_file.endswith(".zip"):
 194                 try:
 195                     vnfd_path = os.path.join(vnf_package_path, "vnfd")
 196                     with zipfile.ZipFile(vendor_pkg_file, 'r') as vendor_zip:
 197                         vender_files = vendor_zip.namelist()
 198                         for vender_file in vender_files:
 199                             if str(vender_file).startswith("Definitions"):
 200                                 vendor_zip.extract(vender_file, vnfd_path)
 201                     with zipfile.ZipFile(vnfd_zip_file, 'w', zipfile.ZIP_DEFLATED) as vnfd_zip:
 202                         def_path = os.path.join(vnfd_path, "Definitions")
 203                         if os.path.exists(def_path):
 204                             def_files = os.listdir(def_path)
 205                             for def_file in def_files:
 206                                 full_path = os.path.join(def_path, def_file)
 207                                 vnfd_zip.write(full_path, def_file)
 208                     return vnfd_zip_file
 209                 except Exception as e:
 210                     logger.error(e)
 211                     if os.path.exists(vnfd_zip):
 212                         os.remove(vnfd_zip)
 213                     raise e
 214                 finally:
 215                     fileutil.delete_dirs(vnfd_path)
 216
 217
 218 class VnfPkgUploadThread(threading.Thread):
 219     """
 220     The Thread for upload VNF pckage
 221     """
 222
 223     def __init__(self, data, vnf_pkg_id):
 224         threading.Thread.__init__(self)
 225         self.vnf_pkg_id = vnf_pkg_id
 226         self.data = data
 227         self.upload_file_name = None
 228
 229     def vnf_pkg_upload_failed_handle(self, error_msg):
 230         logger.error(error_msg)
 231         logger.error(traceback.format_exc())
 232         vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=self.vnf_pkg_id)
 233         if vnf_pkg and vnf_pkg[0].onboardingState == const.PKG_STATUS.UPLOADING:
 234             vnf_pkg.update(onboardingState=const.PKG_STATUS.CREATED)
 235
 236     def run(self):
 237         try:
 238             self.upload_vnf_pkg_from_uri()
 239             parse_vnfd_and_save(self.vnf_pkg_id, self.upload_file_name)
 240         except CatalogException as e:
 241             self.vnf_pkg_upload_failed_handle(e.args[0])
 242         except Exception as e:
 243             self.vnf_pkg_upload_failed_handle(e.args[0])
 244
 245     def upload_vnf_pkg_from_uri(self):
 246         logger.info("Start to upload VNF packge(%s) from URI..." % self.vnf_pkg_id)
 247         vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=self.vnf_pkg_id)
 248         if vnf_pkg[0].onboardingState != const.PKG_STATUS.CREATED:
 249             logger.error("VNF package(%s) is not CREATED" % self.vnf_pkg_id)
 250             raise CatalogException("VNF package (%s) is not created" % self.vnf_pkg_id)
 251         vnf_pkg.update(onboardingState=const.PKG_STATUS.UPLOADING)
 252         send_notification(self.vnf_pkg_id, const.PKG_NOTIFICATION_TYPE.ONBOARDING,
 253                           const.PKG_CHANGE_TYPE.OP_STATE_CHANGE)
 254
 255         uri = ignore_case_get(self.data, "addressInformation")
 256         response = urllib.request.urlopen(uri)
 257
 258         local_file_dir = os.path.join(CATALOG_ROOT_PATH, self.vnf_pkg_id)
 259         self.upload_file_name = os.path.join(local_file_dir, os.path.basename(uri))
 260         if not os.path.exists(local_file_dir):
 261             fileutil.make_dirs(local_file_dir)
 262         with open(self.upload_file_name, "wt") as local_file:
 263             local_file.write(response.read())
 264         response.close()
 265         logger.info('VNF packge(%s) has been uploaded.' % self.vnf_pkg_id)
 266
 267
 268 def get_mfile_data(path):
 269     logger.debug('get_mfile_data path %s' % path)
 270     files = fileutil.filter_files(path, '.mf')
 271     if files:
 272         src_file = os.path.join(path, files[0])
 273         src_dict_list = []
 274         with open(src_file, 'r') as f:
 275             data = f.readlines()
 276             for line in data:
 277                 if line.strip() == "":
 278                     continue
 279                 src_dict = {}
 280                 k, v = line.split(':', maxsplit=1)
 281                 if k.strip() in ["Source", "Algorithm", "Hash"]:
 282                     if k.strip() == "Source" and src_dict:
 283                         src_dict_list.extend(src_dict)
 284                         src_dict = {}
 285                     src_dict[k.strip()] = v.strip()
 286                     print("src_dict:%s" % src_dict)
 287         if src_dict:
 288             src_dict_list.append(src_dict)
 289
 290         logger.debug('get_mfile_data: %s' % src_dict_list)
 291         return src_dict_list
 292
 293
 294 def fill_artifacts_data(vnf_pkg_id):
 295     vnf_pkg_path = os.path.join(CATALOG_ROOT_PATH, vnf_pkg_id)
 296     if os.path.exists(vnf_pkg_path) is False:
 297         return None
 298     files = fileutil.filter_files(vnf_pkg_path, '.csar')
 299     for filename in files:
 300         logger.info('fill_artifacts_data filename (%s)...' % filename)
 301         dst_file_path = os.path.join(vnf_pkg_path, "tmp")
 302         src_file = os.path.join(vnf_pkg_path, filename)
 303         dst_file = os.path.join(dst_file_path, filename)
 304         fileutil.recreate_dir(dst_file_path)
 305         fileutil.copy(src_file, vnf_pkg_path, dst_file)
 306         artifact_vnf_file = fileutil.unzip_file(dst_file, dst_file_path, "")
 307         artifacts = get_mfile_data(artifact_vnf_file)
 308         if artifacts:
 309             return [{
 310                 "artifactPath": artifact.get("Source", ""),
 311                 "checksum": {
 312                     "algorithm": artifact.get("Hash", "Null"),
 313                     "hash": artifact.get("Algorithm", "Null")
 314                 }
 315             } for artifact in artifacts]
 316
 317
 318 def fill_links(pkg_id, is_onboarded=False):
 319     self_href = "/api/vnfpkgm/v1/vnf_packages/%s" % (pkg_id)
 320     links = {
 321         "self": {"href": self_href},
 322         "vnfd": {"href": "%s/%s" % (self_href, "vnfd")},
 323         "packageContent": {"href": "%s/%s" % (self_href, "package_content")}
 324     }
 325     if not is_onboarded:
 326         links.pop("vnfd")
 327     return links
 328
 329
 330 def fill_response_data(nf_pkg):
 331     pkg_info = {}
 332     pkg_info["id"] = nf_pkg.vnfPackageId
 333     pkg_info["vnfdId"] = nf_pkg.vnfdId
 334     pkg_info["vnfProductName"] = nf_pkg.vnfdProductName
 335     pkg_info["vnfSoftwareVersion"] = nf_pkg.vnfSoftwareVersion
 336     pkg_info["vnfdVersion"] = nf_pkg.vnfdVersion
 337     if nf_pkg.checksum:
 338         pkg_info["checksum"] = json.JSONDecoder().decode(nf_pkg.checksum)
 339     pkg_info["softwareImages"] = None  # TODO
 340     pkg_info["additionalArtifacts"] = fill_artifacts_data(nf_pkg.vnfPackageId)
 341     pkg_info["onboardingState"] = nf_pkg.onboardingState
 342     pkg_info["operationalState"] = nf_pkg.operationalState
 343     pkg_info["usageState"] = nf_pkg.usageState
 344     if nf_pkg.userDefinedData:
 345         pkg_info["userDefinedData"] = json.JSONDecoder().decode(nf_pkg.userDefinedData)
 346     pkg_info["_links"] = fill_links(nf_pkg.vnfPackageId, True)
 347     return pkg_info
 348
 349
 350 def parse_vnfd_and_save(vnf_pkg_id, vnf_pkg_path):
 351     logger.info('Start to process VNF package(%s)...' % vnf_pkg_id)
 352     vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 353     vnf_pkg.update(onboardingState=const.PKG_STATUS.PROCESSING)
 354     vnfd_json = toscaparser.parse_vnfd(vnf_pkg_path)
 355     vnfd = json.JSONDecoder().decode(vnfd_json)
 356
 357     if vnfd.get("vnf", "") != "":
 358         vnfd_id = vnfd["vnf"]["properties"].get("descriptor_id", "")
 359         other_pkg = VnfPackageModel.objects.filter(vnfdId=vnfd_id)
 360         if other_pkg and other_pkg[0].vnfPackageId != vnf_pkg_id:
 361             logger.error("VNF package(%s,%s) already exists.", other_pkg[0].vnfPackageId, vnfd_id)
 362             raise CatalogException("VNF package(%s) already exists." % vnfd_id)
 363         vnf_provider = vnfd["vnf"]["properties"].get("provider", "")
 364         vnfd_ver = vnfd["vnf"]["properties"].get("descriptor_version", "")
 365         vnf_software_version = vnfd["vnf"]["properties"].get("software_version", "")
 366         vnfd_product_name = vnfd["vnf"]["properties"].get("product_name", "")
 367         vnf_pkg.update(
 368             vnfPackageId=vnf_pkg_id,
 369             vnfdId=vnfd_id,
 370             vnfdProductName=vnfd_product_name,
 371             vnfVendor=vnf_provider,
 372             vnfdVersion=vnfd_ver,
 373             vnfSoftwareVersion=vnf_software_version,
 374             vnfdModel=vnfd_json,
 375             onboardingState=const.PKG_STATUS.ONBOARDED,
 376             operationalState=const.PKG_STATUS.ENABLED,
 377             usageState=const.PKG_STATUS.NOT_IN_USE,
 378             localFilePath=vnf_pkg_path,
 379             vnfPackageUri=os.path.split(vnf_pkg_path)[-1]
 380         )
 381         send_notification(vnf_pkg_id, const.PKG_NOTIFICATION_TYPE.ONBOARDING,
 382                           const.PKG_CHANGE_TYPE.OP_STATE_CHANGE)
 383     else:
 384         raise CatalogException("VNF propeties and metadata in VNF Package(id=%s) are empty." % vnf_pkg_id)
 385     logger.info('VNF package(%s) has been processed(done).' % vnf_pkg_id)
 386
 387
 388 def handle_upload_failed(vnf_pkg_id):
 389     vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 390     vnf_pkg.update(onboardingState=const.PKG_STATUS.CREATED)
 391
 392
 393 def send_notification(pkg_id, type, pkg_change_type, operational_state=None):
 394     notify = PkgNotifications(type, pkg_id, change_type=pkg_change_type,
 395                               operational_state=operational_state)
 396     notify.send_notification()