catalog/packages/biz/vnf_package.py

   1 # Copyright 2018 ZTE Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import json
  16 import logging
  17 import os
  18 import threading
  19 import traceback
  20 import urllib
  21 import uuid
  22 import zipfile
  23
  24 from catalog.packages import const
  25 from catalog.packages.biz.common import parse_file_range, read, save
  26 from catalog.packages.biz.notificationsutil import PkgNotifications
  27 from catalog.pub.config.config import CATALOG_ROOT_PATH
  28 from catalog.pub.database.models import VnfPackageModel, NSPackageModel
  29 from catalog.pub.exceptions import CatalogException, ResourceNotFoundException
  30 from catalog.pub.utils import fileutil, toscaparser
  31 from catalog.pub.utils.values import ignore_case_get
  32
  33 logger = logging.getLogger(__name__)
  34
  35
  36 class VnfPackage(object):
  37
  38     def __init__(self):
  39         pass
  40
  41     def create_vnf_pkg(self, data):
  42         user_defined_data = ignore_case_get(data, "userDefinedData", {})
  43         vnf_pkg_id = str(uuid.uuid4())
  44         VnfPackageModel.objects.create(
  45             vnfPackageId=vnf_pkg_id,
  46             onboardingState=const.PKG_STATUS.CREATED,
  47             operationalState=const.PKG_STATUS.DISABLED,
  48             usageState=const.PKG_STATUS.NOT_IN_USE,
  49             userDefinedData=json.dumps(user_defined_data)
  50         )
  51         data = {
  52             "id": vnf_pkg_id,
  53             "onboardingState": const.PKG_STATUS.CREATED,
  54             "operationalState": const.PKG_STATUS.DISABLED,
  55             "usageState": const.PKG_STATUS.NOT_IN_USE,
  56             "userDefinedData": user_defined_data,
  57             "_links": None
  58         }
  59         return data
  60
  61     def query_multiple(self):
  62         pkgs_info = []
  63         nf_pkgs = VnfPackageModel.objects.filter()
  64         for nf_pkg in nf_pkgs:
  65             ret = fill_response_data(nf_pkg)
  66             pkgs_info.append(ret)
  67         return pkgs_info
  68
  69     def query_single(self, vnf_pkg_id):
  70         nf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
  71         if not nf_pkg.exists():
  72             logger.error('VNF package(%s) does not exist.' % vnf_pkg_id)
  73             raise ResourceNotFoundException('VNF package(%s) does not exist.' % vnf_pkg_id)
  74         return fill_response_data(nf_pkg[0])
  75
  76     def delete_vnf_pkg(self, vnf_pkg_id):
  77         vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
  78         if not vnf_pkg.exists():
  79             logger.debug('VNF package(%s) has been deleted.' % vnf_pkg_id)
  80             return
  81         '''
  82         if vnf_pkg[0].operationalState != PKG_STATUS.DISABLED:
  83             raise CatalogException("The VNF package (%s) is not disabled" % vnf_pkg_id)
  84         if vnf_pkg[0].usageState != PKG_STATUS.NOT_IN_USE:
  85             raise CatalogException("The VNF package (%s) is in use" % vnf_pkg_id)
  86         '''
  87         del_vnfd_id = vnf_pkg[0].vnfdId
  88         ns_pkgs = NSPackageModel.objects.all()
  89         for ns_pkg in ns_pkgs:
  90             nsd_model = None
  91             if ns_pkg.nsdModel:
  92                 nsd_model = json.JSONDecoder().decode(ns_pkg.nsdModel)
  93             if not nsd_model:
  94                 continue
  95             for vnf in nsd_model['vnfs']:
  96                 if del_vnfd_id == vnf["properties"]["descriptor_id"]:
  97                     raise CatalogException('VNFD(%s) is referenced.' % del_vnfd_id)
  98         vnf_pkg.delete()
  99         send_notification(vnf_pkg_id, const.PKG_NOTIFICATION_TYPE.CHANGE,
 100                           const.PKG_CHANGE_TYPE.PKG_DELETE)
 101
 102         vnf_pkg_path = os.path.join(CATALOG_ROOT_PATH, vnf_pkg_id)
 103         fileutil.delete_dirs(vnf_pkg_path)
 104         logger.info('VNF package(%s) has been deleted.' % vnf_pkg_id)
 105
 106     def upload(self, vnf_pkg_id, remote_file):
 107         logger.info('Start to upload VNF package(%s)...' % vnf_pkg_id)
 108         vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 109         # if vnf_pkg[0].onboardingState != PKG_STATUS.CREATED:
 110         #     logger.error("VNF package(%s) is not CREATED" % vnf_pkg_id)
 111         #     raise CatalogException("VNF package(%s) is not CREATED" % vnf_pkg_id)
 112         vnf_pkg.update(onboardingState=const.PKG_STATUS.UPLOADING)
 113
 114         local_file_name = save(remote_file, vnf_pkg_id)
 115         logger.info('VNF package(%s) has been uploaded.' % vnf_pkg_id)
 116         return local_file_name
 117
 118     def download(self, vnf_pkg_id, file_range):
 119         logger.info('Start to download VNF package(%s)...' % vnf_pkg_id)
 120         nf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 121         if not nf_pkg.exists():
 122             logger.error('VNF package(%s) does not exist.' % vnf_pkg_id)
 123             raise ResourceNotFoundException('VNF package(%s) does not exist.' % vnf_pkg_id)
 124         if nf_pkg[0].onboardingState != const.PKG_STATUS.ONBOARDED:
 125             raise CatalogException("VNF package (%s) is not on-boarded" % vnf_pkg_id)
 126
 127         local_file_path = nf_pkg[0].localFilePath
 128         start, end = parse_file_range(local_file_path, file_range)
 129         logger.info('VNF package (%s) has been downloaded.' % vnf_pkg_id)
 130         return read(local_file_path, start, end)
 131
 132     def download_vnfd(self, vnf_pkg_id):
 133         logger.info('Start to download VNFD of VNF package(%s)...' % vnf_pkg_id)
 134         nf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 135         if not nf_pkg.exists():
 136             logger.error('VNF package(%s) does not exist.' % vnf_pkg_id)
 137             raise ResourceNotFoundException('VNF package(%s) does not exist.' % vnf_pkg_id)
 138         if nf_pkg[0].onboardingState != const.PKG_STATUS.ONBOARDED:
 139             raise CatalogException("VNF package (%s) is not on-boarded" % vnf_pkg_id)
 140
 141         vnfd_zip_file = self.creat_vnfd(vnf_pkg_id, nf_pkg[0].localFilePath)
 142         logger.info('VNFD of VNF package (%s) has been downloaded.' % vnf_pkg_id)
 143         return read(vnfd_zip_file, 0, os.path.getsize(vnfd_zip_file))
 144
 145     def creat_vnfd(self, vnf_pkg_id, vendor_pkg_file):
 146         """
 147         Create VNFD zip file from vendor original package
 148         :param self:
 149         :param vnf_pkg_id: VNF package id (CSAR id)
 150         :param vendor_pkg_file: vendor original package
 151         :return:
 152         """
 153         vnf_package_path = os.path.join(CATALOG_ROOT_PATH, vnf_pkg_id)
 154         if not os.path.exists(vnf_package_path):
 155             os.makedirs(vnf_package_path)
 156         vnfd_zip_file = os.path.join(vnf_package_path, "VNFD.zip")
 157         if os.path.exists(vnfd_zip_file):
 158             return vnfd_zip_file
 159         else:
 160             if vendor_pkg_file.endswith(".csar") or vendor_pkg_file.endswith(".zip"):
 161                 try:
 162                     vnfd_path = os.path.join(vnf_package_path, "vnfd")
 163                     with zipfile.ZipFile(vendor_pkg_file, 'r') as vendor_zip:
 164                         vender_files = vendor_zip.namelist()
 165                         for vender_file in vender_files:
 166                             if str(vender_file).startswith("Definitions"):
 167                                 vendor_zip.extract(vender_file, vnfd_path)
 168                     with zipfile.ZipFile(vnfd_zip_file, 'w', zipfile.ZIP_DEFLATED) as vnfd_zip:
 169                         def_path = os.path.join(vnfd_path, "Definitions")
 170                         if os.path.exists(def_path):
 171                             def_files = os.listdir(def_path)
 172                             for def_file in def_files:
 173                                 full_path = os.path.join(def_path, def_file)
 174                                 vnfd_zip.write(full_path, def_file)
 175                     return vnfd_zip_file
 176                 except Exception as e:
 177                     logger.error(e)
 178                     if os.path.exists(vnfd_zip):
 179                         os.remove(vnfd_zip)
 180                     raise e
 181                 finally:
 182                     fileutil.delete_dirs(vnfd_path)
 183
 184
 185 class VnfPkgUploadThread(threading.Thread):
 186     def __init__(self, data, vnf_pkg_id):
 187         threading.Thread.__init__(self)
 188         self.vnf_pkg_id = vnf_pkg_id
 189         self.data = data
 190         self.upload_file_name = None
 191
 192     def vnf_pkg_upload_failed_handle(self, error_msg):
 193         logger.error(error_msg)
 194         logger.error(traceback.format_exc())
 195         vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=self.vnf_pkg_id)
 196         if vnf_pkg and vnf_pkg[0].onboardingState == const.PKG_STATUS.UPLOADING:
 197             vnf_pkg.update(onboardingState=const.PKG_STATUS.CREATED)
 198
 199     def run(self):
 200         try:
 201             self.upload_vnf_pkg_from_uri()
 202             parse_vnfd_and_save(self.vnf_pkg_id, self.upload_file_name)
 203         except CatalogException as e:
 204             self.vnf_pkg_upload_failed_handle(e.args[0])
 205         except Exception as e:
 206             self.vnf_pkg_upload_failed_handle(e.args[0])
 207
 208     def upload_vnf_pkg_from_uri(self):
 209         logger.info("Start to upload VNF packge(%s) from URI..." % self.vnf_pkg_id)
 210         vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=self.vnf_pkg_id)
 211         if vnf_pkg[0].onboardingState != const.PKG_STATUS.CREATED:
 212             logger.error("VNF package(%s) is not CREATED" % self.vnf_pkg_id)
 213             raise CatalogException("VNF package (%s) is not created" % self.vnf_pkg_id)
 214         vnf_pkg.update(onboardingState=const.PKG_STATUS.UPLOADING)
 215         send_notification(self.vnf_pkg_id, const.PKG_NOTIFICATION_TYPE.ONBOARDING,
 216                           const.PKG_CHANGE_TYPE.OP_STATE_CHANGE)
 217
 218         uri = ignore_case_get(self.data, "addressInformation")
 219         response = urllib.request.urlopen(uri)
 220
 221         local_file_dir = os.path.join(CATALOG_ROOT_PATH, self.vnf_pkg_id)
 222         self.upload_file_name = os.path.join(local_file_dir, os.path.basename(uri))
 223         if not os.path.exists(local_file_dir):
 224             fileutil.make_dirs(local_file_dir)
 225         with open(self.upload_file_name, "wt") as local_file:
 226             local_file.write(response.read())
 227         response.close()
 228         logger.info('VNF packge(%s) has been uploaded.' % self.vnf_pkg_id)
 229
 230
 231 def get_mfile_data(path):
 232     logger.debug('get_mfile_data path %s' % path)
 233     files = fileutil.filter_files(path, '.mf')
 234     if files:
 235         src_file = os.path.join(path, files[0])
 236         src_dict_list = []
 237         with open(src_file, 'r') as f:
 238             data = f.readlines()
 239             for line in data:
 240                 if line.strip() == "":
 241                     continue
 242                 src_dict = {}
 243                 k, v = line.split(':', maxsplit=1)
 244                 if k.strip() in ["Source", "Algorithm", "Hash"]:
 245                     if k.strip() == "Source" and src_dict:
 246                         src_dict_list.extend(src_dict)
 247                         src_dict = {}
 248                     src_dict[k.strip()] = v.strip()
 249                     print("src_dict:%s" % src_dict)
 250         if src_dict:
 251             src_dict_list.append(src_dict)
 252
 253         logger.debug('get_mfile_data: %s' % src_dict_list)
 254         return src_dict_list
 255
 256
 257 def fill_artifacts_data(vnf_pkg_id):
 258     vnf_pkg_path = os.path.join(CATALOG_ROOT_PATH, vnf_pkg_id)
 259     if os.path.exists(vnf_pkg_path) is False:
 260         return None
 261     files = fileutil.filter_files(vnf_pkg_path, '.csar')
 262     for filename in files:
 263         logger.info('fill_artifacts_data filename (%s)...' % filename)
 264         dst_file_path = os.path.join(vnf_pkg_path, "tmp")
 265         src_file = os.path.join(vnf_pkg_path, filename)
 266         dst_file = os.path.join(dst_file_path, filename)
 267         fileutil.recreate_dir(dst_file_path)
 268         fileutil.copy(src_file, vnf_pkg_path, dst_file)
 269         artifact_vnf_file = fileutil.unzip_file(dst_file, dst_file_path, "")
 270         artifacts = get_mfile_data(artifact_vnf_file)
 271         if artifacts:
 272             return [{
 273                 "artifactPath": artifact.get("Source", ""),
 274                 "checksum": {
 275                     "algorithm": artifact.get("Hash", "Null"),
 276                     "hash": artifact.get("Algorithm", "Null")
 277                 }
 278             } for artifact in artifacts]
 279
 280
 281 def fill_links(pkg_id, is_onboarded=False):
 282     self_href = "/api/vnfpkgm/v1/vnf_packages/%s" % (pkg_id)
 283     links = {
 284         "self": {"href": self_href},
 285         "vnfd": {"href": "%s/%s" % (self_href, "vnfd")},
 286         "packageContent": {"href": "%s/%s" % (self_href, "package_content")}
 287     }
 288     if not is_onboarded:
 289         links.pop("vnfd")
 290     return links
 291
 292
 293 def fill_response_data(nf_pkg):
 294     pkg_info = {}
 295     pkg_info["id"] = nf_pkg.vnfPackageId
 296     pkg_info["vnfdId"] = nf_pkg.vnfdId
 297     pkg_info["vnfProductName"] = nf_pkg.vnfdProductName
 298     pkg_info["vnfSoftwareVersion"] = nf_pkg.vnfSoftwareVersion
 299     pkg_info["vnfdVersion"] = nf_pkg.vnfdVersion
 300     if nf_pkg.checksum:
 301         pkg_info["checksum"] = json.JSONDecoder().decode(nf_pkg.checksum)
 302     pkg_info["softwareImages"] = None  # TODO
 303     pkg_info["additionalArtifacts"] = fill_artifacts_data(nf_pkg.vnfPackageId)
 304     pkg_info["onboardingState"] = nf_pkg.onboardingState
 305     pkg_info["operationalState"] = nf_pkg.operationalState
 306     pkg_info["usageState"] = nf_pkg.usageState
 307     if nf_pkg.userDefinedData:
 308         pkg_info["userDefinedData"] = json.JSONDecoder().decode(nf_pkg.userDefinedData)
 309     pkg_info["_links"] = fill_links(nf_pkg.vnfPackageId, True)
 310     return pkg_info
 311
 312
 313 def parse_vnfd_and_save(vnf_pkg_id, vnf_pkg_path):
 314     logger.info('Start to process VNF package(%s)...' % vnf_pkg_id)
 315     vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 316     vnf_pkg.update(onboardingState=const.PKG_STATUS.PROCESSING)
 317     vnfd_json = toscaparser.parse_vnfd(vnf_pkg_path)
 318     vnfd = json.JSONDecoder().decode(vnfd_json)
 319
 320     if vnfd.get("vnf", "") != "":
 321         vnfd_id = vnfd["vnf"]["properties"].get("descriptor_id", "")
 322         other_pkg = VnfPackageModel.objects.filter(vnfdId=vnfd_id)
 323         if other_pkg and other_pkg[0].vnfPackageId != vnf_pkg_id:
 324             logger.error("VNF package(%s,%s) already exists.", other_pkg[0].vnfPackageId, vnfd_id)
 325             raise CatalogException("VNF package(%s) already exists." % vnfd_id)
 326         vnf_provider = vnfd["vnf"]["properties"].get("provider", "")
 327         vnfd_ver = vnfd["vnf"]["properties"].get("descriptor_version", "")
 328         vnf_software_version = vnfd["vnf"]["properties"].get("software_version", "")
 329         vnfd_product_name = vnfd["vnf"]["properties"].get("product_name", "")
 330         vnf_pkg.update(
 331             vnfPackageId=vnf_pkg_id,
 332             vnfdId=vnfd_id,
 333             vnfdProductName=vnfd_product_name,
 334             vnfVendor=vnf_provider,
 335             vnfdVersion=vnfd_ver,
 336             vnfSoftwareVersion=vnf_software_version,
 337             vnfdModel=vnfd_json,
 338             onboardingState=const.PKG_STATUS.ONBOARDED,
 339             operationalState=const.PKG_STATUS.ENABLED,
 340             usageState=const.PKG_STATUS.NOT_IN_USE,
 341             localFilePath=vnf_pkg_path,
 342             vnfPackageUri=os.path.split(vnf_pkg_path)[-1]
 343         )
 344         send_notification(vnf_pkg_id, const.PKG_NOTIFICATION_TYPE.ONBOARDING,
 345                           const.PKG_CHANGE_TYPE.OP_STATE_CHANGE)
 346     else:
 347         raise CatalogException("VNF propeties and metadata in VNF Package(id=%s) are empty." % vnf_pkg_id)
 348     logger.info('VNF package(%s) has been processed(done).' % vnf_pkg_id)
 349
 350
 351 def handle_upload_failed(vnf_pkg_id):
 352     vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
 353     vnf_pkg.update(onboardingState=const.PKG_STATUS.CREATED)
 354
 355
 356 def send_notification(pkg_id, type, pkg_change_type, operational_state=None):
 357     notify = PkgNotifications(type, pkg_id, change_type=pkg_change_type,
 358                               operational_state=operational_state)
 359     notify.send_notification()