lcm/packages/nf_package.py

   1 # Copyright 2016-2017 ZTE Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import json
  16 import logging
  17 import os
  18 import sys
  19 import threading
  20 import time
  21 import traceback
  22 import uuid
  23
  24 from lcm.pub.config.config import IMAGE_ROOT_PATH, IGNORE_DEL_IMG_WEHN_DEL_CSAR
  25 from lcm.pub.database.models import NfPackageModel, VnfPackageFileModel, NfInstModel
  26 from lcm.pub.exceptions import NSLCMException
  27 from lcm.pub.msapi.catalog import P_STATUS_DELETEFAILED, P_STATUS_DELETING
  28 from lcm.pub.msapi.catalog import P_STATUS_NORMAL, P_STATUS_ONBOARDING, P_STATUS_ONBOARDFAILED
  29 from lcm.pub.msapi.catalog import STATUS_ONBOARDED, P_STATUS_ENABLED
  30 from lcm.pub.msapi.catalog import get_download_url_from_catalog
  31 from lcm.pub.msapi.catalog import query_csar_from_catalog, set_csar_state
  32 from lcm.pub.msapi.catalog import query_rawdata_from_catalog, delete_csar_from_catalog
  33 from lcm.pub.msapi.extsys import get_vims
  34 from lcm.pub.nfvi.vim.vimadaptor import VimAdaptor
  35 from lcm.pub.utils import fileutil
  36 from lcm.pub.utils import toscautil
  37 from lcm.pub.utils.jobutil import JobUtil
  38 from lcm.pub.utils.values import ignore_case_get
  39
  40 logger = logging.getLogger(__name__)
  41
  42 SUPPORT_MULTI_VIM, UNSUPPORT_MULTI_VIM = 1, 0
  43 ZTE_PRIVATE = 0
  44 DEPLOY_TYPE_IAAS = 1
  45 IMAGE_FILE = 2
  46 IMAGE_STATUS_ENABLE = "Enable"
  47 MAX_RETRY_TIMES = 300
  48 SLEEP_INTERVAL_SECONDS = 2
  49 IMAGE_ACTIVE = 'active'
  50 JOB_ERROR = 255
  51
  52
  53 class NfOnBoardingThread(threading.Thread):
  54     """
  55     NF package onBoarding
  56     """
  57
  58     def __init__(self, csar_id, vim_ids, lab_vim_id, job_id):
  59         threading.Thread.__init__(self)
  60         self.csar_id = csar_id
  61         self.vim_ids = vim_ids
  62         self.lab_vim_id = lab_vim_id
  63         self.job_id = job_id
  64
  65         self.csar_info = None
  66         self.nfd = None
  67         self.nfd_id = None
  68         self.img_save_path = os.path.join(IMAGE_ROOT_PATH, self.job_id)
  69
  70         self.need_rollback_when_failed = False
  71
  72     def run(self):
  73         try:
  74             self.on_boarding()
  75         except NSLCMException as e:
  76             self.rollback_on_boarding()
  77             JobUtil.add_job_status(self.job_id, JOB_ERROR, e.message)
  78         except:
  79             logger.error(traceback.format_exc())
  80             logger.error(str(sys.exc_info()))
  81             self.rollback_on_boarding()
  82             JobUtil.add_job_status(self.job_id, JOB_ERROR, "Failed to onBoarding CSAR(%s)" % self.csar_id)
  83
  84     def on_boarding(self):
  85         JobUtil.create_job(
  86             inst_type='nf',
  87             jobaction='on_boarding',
  88             inst_id=self.csar_id,
  89             job_id=self.job_id)
  90         JobUtil.add_job_status(self.job_id, 5, "Start CSAR(%s) onBoarding." % self.csar_id)
  91         self.on_boarding_pre_deal()
  92         self.nf_package_save()
  93         self.need_rollback_when_failed = True
  94         nf_images = self.download_nf_images()
  95         self.upload_nf_images(nf_images)
  96         set_csar_state(self.csar_id, "onBoardState", STATUS_ONBOARDED)
  97         set_csar_state(self.csar_id, "processState", P_STATUS_NORMAL)
  98         set_csar_state(self.csar_id, "operationalState", P_STATUS_ENABLED)
  99         JobUtil.add_job_status(self.job_id, 100, "CSAR(%s) onBoarding successfully." % self.csar_id)
 100
 101     def on_boarding_pre_deal(self):
 102         JobUtil.add_job_status(self.job_id, 10, "Check status of CSAR(%s) from catalog." % self.csar_id)
 103
 104         self.csar_info = query_csar_from_catalog(self.csar_id)
 105
 106         on_board_state = ignore_case_get(self.csar_info, "onBoardState")
 107         if on_board_state == STATUS_ONBOARDED:
 108             raise NSLCMException("CSAR(%s) already onBoarded." % self.csar_id)
 109
 110         process_state = ignore_case_get(self.csar_info, "processState")
 111         if process_state == P_STATUS_ONBOARDING:
 112             raise NSLCMException("CSAR(%s) is onBoarding now." % self.csar_id)
 113
 114         JobUtil.add_job_status(self.job_id, 20, "Get model of CSAR(%s) from catalog." % self.csar_id)
 115
 116         raw_data = query_rawdata_from_catalog(self.csar_id)
 117         self.nfd = toscautil.convert_vnfd_model(raw_data["rawData"])  # convert to inner json
 118         self.nfd = json.JSONDecoder().decode(self.nfd)
 119         self.nfd_id = self.nfd["metadata"]["id"]
 120         if NfPackageModel.objects.filter(vnfdid=self.nfd_id):
 121             raise NSLCMException("NFD(%s) already exists." % self.nfd_id)
 122
 123     def nf_package_save(self):
 124         JobUtil.add_job_status(self.job_id, 30, "Save CSAR(%s) to database." % self.csar_id)
 125         vnfd_ver = self.nfd["metadata"].get("vnfd_version")
 126         if not vnfd_ver:
 127             vnfd_ver = self.nfd["metadata"].get("vnfdVersion")
 128         NfPackageModel(
 129             uuid=str(uuid.uuid4()),
 130             nfpackageid=self.csar_id,
 131             vnfdid=self.nfd_id,
 132             vendor=self.nfd["metadata"].get("vendor", "undefined"),
 133             vnfdversion=vnfd_ver,
 134             vnfversion=self.nfd["metadata"].get("version", "undefined"),
 135             vnfdmodel=json.JSONEncoder().encode(self.nfd)
 136         ).save()
 137
 138     def download_nf_images(self):
 139         nf_images = []
 140         for image_file in self.nfd["image_files"]:
 141             img_name = image_file["properties"]["name"]
 142             img_relative_path = image_file["properties"]["file_url"]
 143             img_type = image_file["properties"]["disk_format"]
 144             img_desc = image_file.get("description", "")
 145             img_url, img_local_path = get_download_url_from_catalog(self.csar_id, img_relative_path)
 146             JobUtil.add_job_status(self.job_id, 50, "Start to download Image(%s)." % img_name)
 147             is_download_ok, img_save_full_path = fileutil.download_file_from_http(img_url, self.img_save_path, img_name)
 148             if not is_download_ok:
 149                 raise NSLCMException("Failed to download image from %s" % img_url)
 150             logger.debug("Download Image(%s) to %s successfully.", img_name, img_save_full_path)
 151             nf_images.append({
 152                 "image_url": img_url,
 153                 "img_name": img_name,
 154                 "img_save_full_path": img_save_full_path,
 155                 "img_type": img_type,
 156                 "img_desc": img_desc})
 157         return nf_images
 158
 159     def upload_nf_images(self, nf_images):
 160         vims = get_vims()
 161         if self.lab_vim_id and (not self.vim_ids):
 162             self.vim_ids = [self.lab_vim_id]
 163         for vim_id in self.vim_ids:
 164             sel_vim = [vim for vim in vims if vim["vimId"] == vim_id]
 165             if not sel_vim:
 166                 logger.warn("VIMID(%s) does not exist.", vim_id)
 167                 continue
 168             vim_api = VimAdaptor({
 169                 "vimid": vim_id,
 170                 "vimtype": sel_vim[0]["type"],
 171                 "url": sel_vim[0]["url"],
 172                 "user": sel_vim[0]["userName"],
 173                 "passwd": sel_vim[0]["password"],
 174                 "tenant": sel_vim[0]["tenant"]})
 175             for nf_image in nf_images:
 176                 self.upload_one_nf_image(vim_api, nf_image, vim_id, sel_vim)
 177         fileutil.delete_dirs(self.img_save_path)
 178
 179     def upload_one_nf_image(self, vim_api, nf_image, vim_id, sel_vim):
 180         JobUtil.add_job_status(self.job_id, 80, "Start to upload Image(%s) to VIM(%s)." %
 181                                (nf_image["img_name"], vim_id))
 182         ret = vim_api.create_image({
 183             "image_url": nf_image["image_url"],
 184             "image_name": nf_image["img_name"],
 185             "image_path": nf_image["img_save_full_path"],
 186             "image_type": nf_image["img_type"]})
 187         if ret[0] != 0:
 188             raise NSLCMException("Failed to create image:%s" % ret[1])
 189         image_id = ret[1]["id"]
 190
 191         self.wait_until_upload_done(vim_api, image_id)
 192
 193         VnfPackageFileModel(
 194             vnfpid=self.csar_id,
 195             filename=nf_image["img_name"],
 196             filetype=IMAGE_FILE,
 197             imageid=image_id,
 198             vimid=vim_id,
 199             vimuser=sel_vim[0]["userName"],
 200             tenant=sel_vim[0]["tenant"],
 201             purpose=nf_image["img_desc"],
 202             status=IMAGE_STATUS_ENABLE).save()
 203
 204     def wait_until_upload_done(self, vim_api, image_id):
 205         retry_times = 0
 206         image_create_success = False
 207
 208         while retry_times < MAX_RETRY_TIMES:
 209             retry_times += 1
 210             ret = vim_api.get_image(image_id=image_id)
 211             if ret[0] != 0:
 212                 logging.warn("Failed to query image:%s", ret[1])
 213                 continue
 214             if ret[1]["status"] == IMAGE_ACTIVE:
 215                 image_create_success = True
 216                 break
 217             time.sleep(SLEEP_INTERVAL_SECONDS)
 218
 219         if not image_create_success:
 220             timeout_seconds = MAX_RETRY_TIMES * SLEEP_INTERVAL_SECONDS
 221             raise NSLCMException("Failed to create image:timeout(%s seconds.)" % timeout_seconds)
 222
 223     def rollback_on_boarding(self):
 224         if not self.need_rollback_when_failed:
 225             return
 226         try:
 227             set_csar_state(self.csar_id, "processState", P_STATUS_ONBOARDFAILED)
 228             NfPackageModel.objects.filter(nfpackageid=self.csar_id).delete()
 229             VnfPackageFileModel.objects.filter(vnfpid=self.csar_id).delete()
 230             fileutil.delete_dirs(self.img_save_path)
 231         except:
 232             logger.error(traceback.format_exc())
 233             logger.error(str(sys.exc_info()))
 234
 235
 236 ######################################################################################################################
 237
 238
 239 class NfPkgDeleteThread(threading.Thread):
 240     """
 241     NF Package Deleting
 242     """
 243
 244     def __init__(self, csar_id, job_id):
 245         threading.Thread.__init__(self)
 246         self.csar_id = csar_id
 247         self.job_id = job_id
 248
 249     def run(self):
 250         try:
 251             self.delete_csar()
 252         except NSLCMException as e:
 253             set_csar_state(self.csar_id, "processState", P_STATUS_DELETEFAILED)
 254             JobUtil.add_job_status(self.job_id, JOB_ERROR, e.message)
 255         except:
 256             logger.error(traceback.format_exc())
 257             logger.error(str(sys.exc_info()))
 258             set_csar_state(self.csar_id, "processState", P_STATUS_DELETEFAILED)
 259             JobUtil.add_job_status(self.job_id, JOB_ERROR, "Failed to delete CSAR(%s)" % self.csar_id)
 260
 261     def delete_csar(self):
 262         JobUtil.create_job(
 263             inst_type='nf',
 264             jobaction='delete',
 265             inst_id=self.csar_id,
 266             job_id=self.job_id)
 267         JobUtil.add_job_status(self.job_id, 5, "Start to delete CSAR(%s)." % self.csar_id)
 268         if query_csar_from_catalog(self.csar_id, "processState") == P_STATUS_DELETING:
 269             JobUtil.add_job_status(self.job_id, 100, "CSAR(%s) is deleting now." % self.csar_id)
 270             return
 271
 272         if NfInstModel.objects.filter(package_id=self.csar_id):
 273             ret = set_csar_state(self.csar_id, "deletionPending", True)
 274             JobUtil.add_job_status(self.job_id, 100, ret[1])
 275             return
 276
 277         NfPackage().delete_csar(self.csar_id, self.job_id)
 278
 279
 280 class NfPkgDeletePendingThread(threading.Thread):
 281     """
 282     NF Package Delete Pending
 283     """
 284
 285     def __init__(self, csar_id, job_id):
 286         threading.Thread.__init__(self)
 287         self.csar_id = csar_id
 288         self.job_id = job_id
 289
 290     def run(self):
 291         try:
 292             self.delete_pending_csar()
 293         except NSLCMException as e:
 294             set_csar_state(self.csar_id, "processState", P_STATUS_DELETEFAILED)
 295             JobUtil.add_job_status(self.job_id, JOB_ERROR, e.message)
 296         except:
 297             logger.error(traceback.format_exc())
 298             logger.error(str(sys.exc_info()))
 299             set_csar_state(self.csar_id, "processState", P_STATUS_DELETEFAILED)
 300             JobUtil.add_job_status(self.job_id, JOB_ERROR, "Failed to delete CSAR(%s)" % self.csar_id)
 301
 302     def delete_pending_csar(self):
 303         JobUtil.create_job(
 304             inst_type='nf',
 305             jobaction='delete_pending',
 306             inst_id=self.csar_id,
 307             job_id=self.job_id)
 308         JobUtil.add_job_status(self.job_id, 5, "Start to delete pending CSAR(%s)." % self.csar_id)
 309
 310         if not NfPackageModel.objects.filter(nfpackageid=self.csar_id):
 311             JobUtil.add_job_status(self.job_id, 100, "Delete pending CSAR(%s) successfully." % self.csar_id)
 312             return
 313
 314         csar_info = query_csar_from_catalog(self.csar_id)
 315
 316         process_state = ignore_case_get(csar_info, "processState")
 317         if process_state == P_STATUS_DELETING:
 318             JobUtil.add_job_status(self.job_id, 100, "CSAR(%s) is deleting now." % self.csar_id)
 319             return
 320
 321         deletion_pending = ignore_case_get(csar_info, "deletionPending")
 322         if deletion_pending.lower() == "false":
 323             JobUtil.add_job_status(self.job_id, 100, "CSAR(%s) need not to be deleted." % self.csar_id)
 324             return
 325
 326         if NfInstModel.objects.filter(package_id=self.csar_id):
 327             JobUtil.add_job_status(self.job_id, 100, "CSAR(%s) is in using, cannot be deleted." % self.csar_id)
 328             return
 329
 330         NfPackage().delete_csar(self.csar_id, self.job_id)
 331
 332
 333 ####################################################################################################################
 334 class NfPackage(object):
 335     """
 336     Actions for nf package.
 337     """
 338
 339     def __init__(self):
 340         pass
 341
 342     def get_csars(self):
 343         ret = {"csars": []}
 344         nf_pkgs = NfPackageModel.objects.filter()
 345         for nf_pkg in nf_pkgs:
 346             ret["csars"].append({
 347                 "csarId": nf_pkg.nfpackageid,
 348                 "vnfdId": nf_pkg.vnfdid
 349             })
 350         return ret
 351
 352     def get_csar(self, csar_id):
 353         pkg_info = {}
 354         nf_pkg = NfPackageModel.objects.filter(nfpackageid=csar_id)
 355         if nf_pkg:
 356             pkg_info["vnfdId"] = nf_pkg[0].vnfdid
 357             pkg_info["vnfdProvider"] = nf_pkg[0].vendor
 358             pkg_info["vnfdVersion"] = nf_pkg[0].vnfdversion
 359             pkg_info["vnfVersion"] = nf_pkg[0].vnfversion
 360
 361         casrinfo = query_csar_from_catalog(csar_id)
 362         props_of_catalog = [
 363             "name", "provider", "version", "operationalState", "usageState",
 364             "onBoardState", "processState", "deletionPending", "downloadUri",
 365             "createTime", "modifyTime", "format", "size"]
 366         for prop in props_of_catalog:
 367             pkg_info[prop] = ignore_case_get(casrinfo, prop)
 368
 369         nf_pkg_files = VnfPackageFileModel.objects.filter(vnfpid=csar_id)
 370         img_info = [{
 371             "index": str(i),
 372             "fileName": nf_pkg_files[i].filename,
 373             "imageId": nf_pkg_files[i].imageid,
 374             "vimId": nf_pkg_files[i].vimid,
 375             "vimUser": nf_pkg_files[i].vimuser,
 376             "tenant": nf_pkg_files[i].tenant,
 377             "status": nf_pkg_files[i].status}
 378             for i in range(len(nf_pkg_files))]
 379
 380         vnf_insts = NfInstModel.objects.filter(package_id=csar_id)
 381         vnf_inst_info = [{"vnfInstanceId": vnf_inst.nfinstid,
 382                           "vnfInstanceName": vnf_inst.nf_name} for vnf_inst in vnf_insts]
 383
 384         return [0, {"csarId": csar_id,
 385                     "packageInfo": pkg_info,
 386                     "imageInfo": img_info,
 387                     "vnfInstanceInfo": vnf_inst_info}]
 388
 389     def delete_csar(self, csar_id, job_id):
 390         JobUtil.add_job_status(job_id, 10, "Set processState of CSAR(%s)." % csar_id)
 391         set_csar_state(csar_id, "processState", P_STATUS_DELETING)
 392
 393         JobUtil.add_job_status(job_id, 20, "Get package files of CSAR(%s)." % csar_id)
 394         all_nf_pkg_files = VnfPackageFileModel.objects.all()
 395         nf_pkg_files = VnfPackageFileModel.objects.filter(vnfpid=csar_id)
 396         vims = get_vims()
 397
 398         for pkg_file in nf_pkg_files:
 399             if IGNORE_DEL_IMG_WEHN_DEL_CSAR:
 400                 logger.warn("ignore delete image(%s)" % pkg_file.filename)
 401                 continue
 402             JobUtil.add_job_status(job_id, 50, "Delete image(%s) of CSAR(%s)." %
 403                                    (pkg_file.filename, csar_id))
 404             if self.is_image_refed_by_other_nf_pkg(all_nf_pkg_files, pkg_file.imageid, csar_id, pkg_file.vimid):
 405                 logger.warn("Image(%s) is refered by CSAR(%s).", pkg_file.filename, csar_id)
 406                 continue
 407             sel_vim = [vim for vim in vims if vim["vimId"] == pkg_file.vimid]
 408             if not sel_vim:
 409                 logger.warn("Vim(%s) does not exist.", pkg_file.vimid)
 410                 continue
 411             vim_api = VimAdaptor({
 412                 "vimid": pkg_file.vimid,
 413                 "vimtype": sel_vim[0]["type"],
 414                 "url": sel_vim[0]["url"],
 415                 "user": sel_vim[0]["userName"],
 416                 "passwd": sel_vim[0]["password"],
 417                 "tenant": sel_vim[0]["tenant"]})
 418             ret = vim_api.delete_image(pkg_file.imageid)
 419             if ret[0] != 0:
 420                 logger.error("Failed to delete image(%s) from vim(%s)", pkg_file.filename, pkg_file.vimid)
 421
 422         JobUtil.add_job_status(job_id, 70, "Delete CSAR(%s) from catalog." % csar_id)
 423         ret = delete_csar_from_catalog(csar_id)
 424         if ret[0] != 0:
 425             raise NSLCMException(ret[1])
 426
 427         JobUtil.add_job_status(job_id, 90, "Delete CSAR(%s) from database." % csar_id)
 428         VnfPackageFileModel.objects.filter(vnfpid=csar_id).delete()
 429         NfPackageModel.objects.filter(nfpackageid=csar_id).delete()
 430
 431         JobUtil.add_job_status(job_id, 100, "Delete CSAR(%s) successfully." % csar_id)
 432
 433     def is_image_refed_by_other_nf_pkg(self, nf_pkg_files, imageid, csar_id, vim_id):
 434         for f in nf_pkg_files:
 435             if f.imageid == imageid and f.vimid == vim_id and f.vnfpid != csar_id:
 436                 return True
 437         return False