1 # Copyright 2018 ZTE Corporation.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
24 from catalog.packages import const
25 from catalog.packages.biz.common import parse_file_range, read, save
26 from catalog.packages.biz.notificationsutil import PkgNotifications
27 from catalog.pub.config.config import CATALOG_ROOT_PATH
28 from catalog.pub.database.models import VnfPackageModel, NSPackageModel
29 from catalog.pub.exceptions import CatalogException, ResourceNotFoundException
30 from catalog.pub.utils import fileutil, toscaparser
31 from catalog.pub.utils.values import ignore_case_get
33 logger = logging.getLogger(__name__)
36 class VnfPackage(object):
38 The class for VNF package management
44 def create_vnf_pkg(self, data):
47 :param data: user defined data
48 :return: VNF package info
50 user_defined_data = ignore_case_get(data, "userDefinedData", {})
51 vnf_pkg_id = str(uuid.uuid4())
52 VnfPackageModel.objects.create(
53 vnfPackageId=vnf_pkg_id,
54 onboardingState=const.PKG_STATUS.CREATED,
55 operationalState=const.PKG_STATUS.DISABLED,
56 usageState=const.PKG_STATUS.NOT_IN_USE,
57 userDefinedData=json.dumps(user_defined_data)
61 "onboardingState": const.PKG_STATUS.CREATED,
62 "operationalState": const.PKG_STATUS.DISABLED,
63 "usageState": const.PKG_STATUS.NOT_IN_USE,
64 "userDefinedData": user_defined_data,
69 def query_multiple(self):
71 Query the list of VNF package
72 :return: The list of VNF pakcage
75 nf_pkgs = VnfPackageModel.objects.filter()
76 for nf_pkg in nf_pkgs:
77 ret = fill_response_data(nf_pkg)
81 def query_single(self, vnf_pkg_id):
83 Query a single VNF package by given id
84 :param vnf_pkg_id: The id of VNF package
85 :return: VNF pckage info
87 nf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
88 if not nf_pkg.exists():
89 logger.error('VNF package(%s) does not exist.' % vnf_pkg_id)
90 raise ResourceNotFoundException('VNF package(%s) does not exist.' % vnf_pkg_id)
91 return fill_response_data(nf_pkg[0])
93 def delete_vnf_pkg(self, vnf_pkg_id):
95 Delete a VNF package by give id
96 :param vnf_pkg_id: The id of VNF package
99 vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
100 if not vnf_pkg.exists():
101 logger.debug('VNF package(%s) has been deleted.' % vnf_pkg_id)
104 if vnf_pkg[0].operationalState != PKG_STATUS.DISABLED:
105 raise CatalogException("The VNF package (%s) is not disabled" % vnf_pkg_id)
106 if vnf_pkg[0].usageState != PKG_STATUS.NOT_IN_USE:
107 raise CatalogException("The VNF package (%s) is in use" % vnf_pkg_id)
109 del_vnfd_id = vnf_pkg[0].vnfdId
110 ns_pkgs = NSPackageModel.objects.all()
111 for ns_pkg in ns_pkgs:
114 nsd_model = json.JSONDecoder().decode(ns_pkg.nsdModel)
117 for vnf in nsd_model['vnfs']:
118 if del_vnfd_id == vnf["properties"]["descriptor_id"]:
119 raise CatalogException('VNFD(%s) is referenced.' % del_vnfd_id)
121 send_notification(vnf_pkg_id, const.PKG_NOTIFICATION_TYPE.CHANGE,
122 const.PKG_CHANGE_TYPE.PKG_DELETE)
124 vnf_pkg_path = os.path.join(CATALOG_ROOT_PATH, vnf_pkg_id)
125 fileutil.delete_dirs(vnf_pkg_path)
126 logger.info('VNF package(%s) has been deleted.' % vnf_pkg_id)
128 def upload(self, vnf_pkg_id, remote_file):
130 Update VNF pckage file for given id
131 :param vnf_pkg_id: The id of VNF package
132 :param remote_file: VNF package file
135 logger.info('Start to upload VNF package(%s)...' % vnf_pkg_id)
136 vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
137 # if vnf_pkg[0].onboardingState != PKG_STATUS.CREATED:
138 # logger.error("VNF package(%s) is not CREATED" % vnf_pkg_id)
139 # raise CatalogException("VNF package(%s) is not CREATED" % vnf_pkg_id)
140 vnf_pkg.update(onboardingState=const.PKG_STATUS.UPLOADING)
142 local_file_name = save(remote_file, vnf_pkg_id)
143 logger.info('VNF package(%s) has been uploaded.' % vnf_pkg_id)
144 return local_file_name
146 def download(self, vnf_pkg_id, file_range):
147 logger.info('Start to download VNF package(%s)...' % vnf_pkg_id)
148 nf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
149 if not nf_pkg.exists():
150 logger.error('VNF package(%s) does not exist.' % vnf_pkg_id)
151 raise ResourceNotFoundException('VNF package(%s) does not exist.' % vnf_pkg_id)
152 if nf_pkg[0].onboardingState != const.PKG_STATUS.ONBOARDED:
153 raise CatalogException("VNF package (%s) is not on-boarded" % vnf_pkg_id)
155 local_file_path = nf_pkg[0].localFilePath
156 start, end = parse_file_range(local_file_path, file_range)
157 logger.info('VNF package (%s) has been downloaded.' % vnf_pkg_id)
158 return read(local_file_path, start, end)
160 def download_vnfd(self, vnf_pkg_id):
162 Download VNFD for given id
163 :param vnf_pkg_id: The id of VNF package
166 logger.info('Start to download VNFD of VNF package(%s)...' % vnf_pkg_id)
167 nf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
168 if not nf_pkg.exists():
169 logger.error('VNF package(%s) does not exist.' % vnf_pkg_id)
170 raise ResourceNotFoundException('VNF package(%s) does not exist.' % vnf_pkg_id)
171 if nf_pkg[0].onboardingState != const.PKG_STATUS.ONBOARDED:
172 raise CatalogException("VNF package (%s) is not on-boarded" % vnf_pkg_id)
174 vnfd_zip_file = self.creat_vnfd(vnf_pkg_id, nf_pkg[0].localFilePath)
175 logger.info('VNFD of VNF package (%s) has been downloaded.' % vnf_pkg_id)
176 return read(vnfd_zip_file, 0, os.path.getsize(vnfd_zip_file))
178 def creat_vnfd(self, vnf_pkg_id, vendor_pkg_file):
180 Create VNFD zip file from vendor original package
182 :param vnf_pkg_id: VNF package id (CSAR id)
183 :param vendor_pkg_file: vendor original package
186 vnf_package_path = os.path.join(CATALOG_ROOT_PATH, vnf_pkg_id)
187 if not os.path.exists(vnf_package_path):
188 os.makedirs(vnf_package_path)
189 vnfd_zip_file = os.path.join(vnf_package_path, "VNFD.zip")
190 if os.path.exists(vnfd_zip_file):
193 if vendor_pkg_file.endswith(".csar") or vendor_pkg_file.endswith(".zip"):
195 vnfd_path = os.path.join(vnf_package_path, "vnfd")
196 with zipfile.ZipFile(vendor_pkg_file, 'r') as vendor_zip:
197 vender_files = vendor_zip.namelist()
198 for vender_file in vender_files:
199 if str(vender_file).startswith("Definitions"):
200 vendor_zip.extract(vender_file, vnfd_path)
201 with zipfile.ZipFile(vnfd_zip_file, 'w', zipfile.ZIP_DEFLATED) as vnfd_zip:
202 def_path = os.path.join(vnfd_path, "Definitions")
203 if os.path.exists(def_path):
204 def_files = os.listdir(def_path)
205 for def_file in def_files:
206 full_path = os.path.join(def_path, def_file)
207 vnfd_zip.write(full_path, def_file)
209 except Exception as e:
211 if os.path.exists(vnfd_zip):
215 fileutil.delete_dirs(vnfd_path)
218 class VnfPkgUploadThread(threading.Thread):
220 The Thread for upload VNF pckage
223 def __init__(self, data, vnf_pkg_id):
224 threading.Thread.__init__(self)
225 self.vnf_pkg_id = vnf_pkg_id
227 self.upload_file_name = None
229 def vnf_pkg_upload_failed_handle(self, error_msg):
230 logger.error(error_msg)
231 logger.error(traceback.format_exc())
232 vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=self.vnf_pkg_id)
233 if vnf_pkg and vnf_pkg[0].onboardingState == const.PKG_STATUS.UPLOADING:
234 vnf_pkg.update(onboardingState=const.PKG_STATUS.CREATED)
238 self.upload_vnf_pkg_from_uri()
239 parse_vnfd_and_save(self.vnf_pkg_id, self.upload_file_name)
240 except CatalogException as e:
241 self.vnf_pkg_upload_failed_handle(e.args[0])
242 except Exception as e:
243 self.vnf_pkg_upload_failed_handle(e.args[0])
245 def upload_vnf_pkg_from_uri(self):
246 logger.info("Start to upload VNF packge(%s) from URI..." % self.vnf_pkg_id)
247 vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=self.vnf_pkg_id)
248 if vnf_pkg[0].onboardingState != const.PKG_STATUS.CREATED:
249 logger.error("VNF package(%s) is not CREATED" % self.vnf_pkg_id)
250 raise CatalogException("VNF package (%s) is not created" % self.vnf_pkg_id)
251 vnf_pkg.update(onboardingState=const.PKG_STATUS.UPLOADING)
252 send_notification(self.vnf_pkg_id, const.PKG_NOTIFICATION_TYPE.ONBOARDING,
253 const.PKG_CHANGE_TYPE.OP_STATE_CHANGE)
255 uri = ignore_case_get(self.data, "addressInformation")
256 response = urllib.request.urlopen(uri)
258 local_file_dir = os.path.join(CATALOG_ROOT_PATH, self.vnf_pkg_id)
259 self.upload_file_name = os.path.join(local_file_dir, os.path.basename(uri))
260 if not os.path.exists(local_file_dir):
261 fileutil.make_dirs(local_file_dir)
262 with open(self.upload_file_name, "wt") as local_file:
263 local_file.write(response.read())
265 logger.info('VNF packge(%s) has been uploaded.' % self.vnf_pkg_id)
268 def get_mfile_data(path):
269 logger.debug('get_mfile_data path %s' % path)
270 files = fileutil.filter_files(path, '.mf')
272 src_file = os.path.join(path, files[0])
274 with open(src_file, 'r') as f:
277 if line.strip() == "":
280 k, v = line.split(':', maxsplit=1)
281 if k.strip() in ["Source", "Algorithm", "Hash"]:
282 if k.strip() == "Source" and src_dict:
283 src_dict_list.extend(src_dict)
285 src_dict[k.strip()] = v.strip()
286 print("src_dict:%s" % src_dict)
288 src_dict_list.append(src_dict)
290 logger.debug('get_mfile_data: %s' % src_dict_list)
294 def fill_artifacts_data(vnf_pkg_id):
295 vnf_pkg_path = os.path.join(CATALOG_ROOT_PATH, vnf_pkg_id)
296 if os.path.exists(vnf_pkg_path) is False:
298 files = fileutil.filter_files(vnf_pkg_path, '.csar')
299 for filename in files:
300 logger.info('fill_artifacts_data filename (%s)...' % filename)
301 dst_file_path = os.path.join(vnf_pkg_path, "tmp")
302 src_file = os.path.join(vnf_pkg_path, filename)
303 dst_file = os.path.join(dst_file_path, filename)
304 fileutil.recreate_dir(dst_file_path)
305 fileutil.copy(src_file, vnf_pkg_path, dst_file)
306 artifact_vnf_file = fileutil.unzip_file(dst_file, dst_file_path, "")
307 artifacts = get_mfile_data(artifact_vnf_file)
310 "artifactPath": artifact.get("Source", ""),
312 "algorithm": artifact.get("Hash", "Null"),
313 "hash": artifact.get("Algorithm", "Null")
315 } for artifact in artifacts]
318 def fill_links(pkg_id, is_onboarded=False):
319 self_href = "/api/vnfpkgm/v1/vnf_packages/%s" % (pkg_id)
321 "self": {"href": self_href},
322 "vnfd": {"href": "%s/%s" % (self_href, "vnfd")},
323 "packageContent": {"href": "%s/%s" % (self_href, "package_content")}
330 def fill_response_data(nf_pkg):
332 pkg_info["id"] = nf_pkg.vnfPackageId
333 pkg_info["vnfdId"] = nf_pkg.vnfdId
334 pkg_info["vnfProductName"] = nf_pkg.vnfdProductName
335 pkg_info["vnfSoftwareVersion"] = nf_pkg.vnfSoftwareVersion
336 pkg_info["vnfdVersion"] = nf_pkg.vnfdVersion
338 pkg_info["checksum"] = json.JSONDecoder().decode(nf_pkg.checksum)
339 pkg_info["softwareImages"] = None # TODO
340 pkg_info["additionalArtifacts"] = fill_artifacts_data(nf_pkg.vnfPackageId)
341 pkg_info["onboardingState"] = nf_pkg.onboardingState
342 pkg_info["operationalState"] = nf_pkg.operationalState
343 pkg_info["usageState"] = nf_pkg.usageState
344 if nf_pkg.userDefinedData:
345 pkg_info["userDefinedData"] = json.JSONDecoder().decode(nf_pkg.userDefinedData)
346 pkg_info["_links"] = fill_links(nf_pkg.vnfPackageId, True)
350 def parse_vnfd_and_save(vnf_pkg_id, vnf_pkg_path):
351 logger.info('Start to process VNF package(%s)...' % vnf_pkg_id)
352 vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
353 vnf_pkg.update(onboardingState=const.PKG_STATUS.PROCESSING)
354 vnfd_json = toscaparser.parse_vnfd(vnf_pkg_path)
355 vnfd = json.JSONDecoder().decode(vnfd_json)
357 if vnfd.get("vnf", "") != "":
358 vnfd_id = vnfd["vnf"]["properties"].get("descriptor_id", "")
359 other_pkg = VnfPackageModel.objects.filter(vnfdId=vnfd_id)
360 if other_pkg and other_pkg[0].vnfPackageId != vnf_pkg_id:
361 logger.error("VNF package(%s,%s) already exists.", other_pkg[0].vnfPackageId, vnfd_id)
362 raise CatalogException("VNF package(%s) already exists." % vnfd_id)
363 vnf_provider = vnfd["vnf"]["properties"].get("provider", "")
364 vnfd_ver = vnfd["vnf"]["properties"].get("descriptor_version", "")
365 vnf_software_version = vnfd["vnf"]["properties"].get("software_version", "")
366 vnfd_product_name = vnfd["vnf"]["properties"].get("product_name", "")
368 vnfPackageId=vnf_pkg_id,
370 vnfdProductName=vnfd_product_name,
371 vnfVendor=vnf_provider,
372 vnfdVersion=vnfd_ver,
373 vnfSoftwareVersion=vnf_software_version,
375 onboardingState=const.PKG_STATUS.ONBOARDED,
376 operationalState=const.PKG_STATUS.ENABLED,
377 usageState=const.PKG_STATUS.NOT_IN_USE,
378 localFilePath=vnf_pkg_path,
379 vnfPackageUri=os.path.split(vnf_pkg_path)[-1]
381 send_notification(vnf_pkg_id, const.PKG_NOTIFICATION_TYPE.ONBOARDING,
382 const.PKG_CHANGE_TYPE.OP_STATE_CHANGE)
384 raise CatalogException("VNF propeties and metadata in VNF Package(id=%s) are empty." % vnf_pkg_id)
385 logger.info('VNF package(%s) has been processed(done).' % vnf_pkg_id)
388 def handle_upload_failed(vnf_pkg_id):
389 vnf_pkg = VnfPackageModel.objects.filter(vnfPackageId=vnf_pkg_id)
390 vnf_pkg.update(onboardingState=const.PKG_STATUS.CREATED)
393 def send_notification(pkg_id, type, pkg_change_type, operational_state=None):
394 notify = PkgNotifications(type, pkg_id, change_type=pkg_change_type,
395 operational_state=operational_state)
396 notify.send_notification()