gerrit.onap Code Review - ccsdk/cds.git/blob

   1 /*
   2  * Copyright © 2017-2018 AT&T Intellectual Property.
   3  * Modifications Copyright © 2019 IBM.
   4  * Modifications Copyright © 2020 Orange.
   5  * Modifications Copyright © 2020 Deutsche Telekom AG.
   6  *
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *     http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  */
  19
  20 package org.onap.ccsdk.cds.blueprintsprocessor.functions.k8s.profile.upload
  21
  22 import com.fasterxml.jackson.databind.JsonNode
  23 import com.fasterxml.jackson.databind.node.ObjectNode
  24 import org.apache.commons.io.FileUtils
  25 import org.onap.ccsdk.cds.blueprintsprocessor.core.BluePrintPropertiesService
  26 import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.ExecutionServiceInput
  27 import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.ResourceResolutionConstants
  28 import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.ResourceResolutionService
  29 import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.AbstractComponentFunction
  30 import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintConstants
  31 import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintProcessorException
  32 import org.onap.ccsdk.cds.controllerblueprints.core.asJsonNode
  33 import org.onap.ccsdk.cds.controllerblueprints.core.data.ArtifactDefinition
  34 import org.onap.ccsdk.cds.controllerblueprints.core.returnNullIfMissing
  35 import org.onap.ccsdk.cds.controllerblueprints.core.service.BluePrintVelocityTemplateService
  36 import org.onap.ccsdk.cds.controllerblueprints.core.utils.ArchiveType
  37 import org.onap.ccsdk.cds.controllerblueprints.core.utils.BluePrintArchiveUtils
  38 import org.onap.ccsdk.cds.controllerblueprints.core.utils.JacksonUtils
  39 import org.slf4j.LoggerFactory
  40 import org.springframework.beans.factory.config.ConfigurableBeanFactory
  41 import org.springframework.context.annotation.Scope
  42 import org.springframework.stereotype.Component
  43 import org.yaml.snakeyaml.Yaml
  44 import java.io.File
  45 import java.nio.file.Files
  46 import java.nio.file.Path
  47 import java.nio.file.Paths
  48 import kotlin.collections.ArrayList
  49
  50 @Component("component-k8s-profile-upload")
  51 @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
  52 open class K8sProfileUploadComponent(
  53     private var bluePrintPropertiesService: BluePrintPropertiesService,
  54     private val resourceResolutionService: ResourceResolutionService
  55 ) :
  56
  57     AbstractComponentFunction() {
  58
  59     companion object {
  60         const val INPUT_K8S_PROFILE_NAME = "k8s-rb-profile-name"
  61         const val INPUT_K8S_DEFINITION_NAME = "k8s-rb-definition-name"
  62         const val INPUT_K8S_DEFINITION_VERSION = "k8s-rb-definition-version"
  63         const val INPUT_K8S_PROFILE_NAMESPACE = "k8s-rb-profile-namespace"
  64         const val INPUT_K8S_PROFILE_SOURCE = "k8s-rb-profile-source"
  65         const val INPUT_RESOURCE_ASSIGNMENT_MAP = "resource-assignment-map"
  66
  67         const val OUTPUT_STATUSES = "statuses"
  68         const val OUTPUT_SKIPPED = "skipped"
  69         const val OUTPUT_UPLOADED = "uploaded"
  70         const val OUTPUT_ERROR = "error"
  71     }
  72
  73     private val log = LoggerFactory.getLogger(K8sProfileUploadComponent::class.java)!!
  74
  75     override suspend fun processNB(executionRequest: ExecutionServiceInput) {
  76         log.info("Triggering K8s Profile Upload component logic.")
  77
  78         val inputParameterNames = arrayOf(
  79             INPUT_K8S_PROFILE_NAME,
  80             INPUT_K8S_DEFINITION_NAME,
  81             INPUT_K8S_DEFINITION_VERSION,
  82             INPUT_K8S_PROFILE_NAMESPACE,
  83             INPUT_K8S_PROFILE_SOURCE
  84         )
  85         var outputPrefixStatuses = mutableMapOf<String, String>()
  86         var inputParamsMap = mutableMapOf<String, JsonNode?>()
  87
  88         inputParameterNames.forEach {
  89             inputParamsMap[it] = getOptionalOperationInput(it)?.returnNullIfMissing()
  90         }
  91
  92         log.info("Getting the template prefixes")
  93         val prefixList: ArrayList<String> = getTemplatePrefixList(executionRequest)
  94
  95         log.info("Iterating over prefixes in resource assignment map.")
  96         for (prefix in prefixList) {
  97             // Prefilling prefix sucess status
  98             outputPrefixStatuses.put(prefix, OUTPUT_SKIPPED)
  99             // Resource assignment map is organized by prefixes, in each iteraton we work only
 100             // on one section of resource assignment map
 101             val prefixNode: JsonNode = operationInputs[INPUT_RESOURCE_ASSIGNMENT_MAP]?.get(prefix) ?: continue
 102             val assignmentMapPrefix = JacksonUtils.jsonNode(prefixNode.toPrettyString()) as ObjectNode
 103
 104             // We are copying the map because for each prefix it might be completed with a different data
 105             var prefixInputParamsMap = inputParamsMap.toMutableMap()
 106             prefixInputParamsMap.forEach { (inputParamName, value) ->
 107                 if (value == null) {
 108                     val mapValue = assignmentMapPrefix?.get(inputParamName)
 109                     log.debug("$inputParamName value was $value so we fetch $mapValue")
 110                     prefixInputParamsMap[inputParamName] = mapValue
 111                 }
 112             }
 113
 114             // For clarity we pull out the required fields
 115             val profileName: String? = prefixInputParamsMap[INPUT_K8S_PROFILE_NAME]?.returnNullIfMissing()?.asText()
 116             val definitionName: String? = prefixInputParamsMap[INPUT_K8S_DEFINITION_NAME]?.returnNullIfMissing()?.asText()
 117             val definitionVersion: String? = prefixInputParamsMap[INPUT_K8S_DEFINITION_VERSION]?.returnNullIfMissing()?.asText()
 118
 119             val k8sProfileUploadConfiguration = K8sProfileUploadConfiguration(bluePrintPropertiesService)
 120
 121             // Creating API connector
 122             var api = K8sPluginApi(
 123                 k8sProfileUploadConfiguration.getProperties().username,
 124                 k8sProfileUploadConfiguration.getProperties().password,
 125                 k8sProfileUploadConfiguration.getProperties().url,
 126                 definitionName,
 127                 definitionVersion
 128             )
 129
 130             if ((profileName == null) || (definitionName == null) || (definitionVersion == null)) {
 131                 log.warn("Prefix $prefix does not have required data for us to continue.")
 132             } else if (!api.hasDefinition()) {
 133                 log.warn("K8s RB Definition ($definitionName/$definitionVersion) not found ")
 134             } else if (profileName == "") {
 135                 log.warn("K8s rb profile name is empty! Either define profile name to use or choose default")
 136             } else if (api.hasProfile(profileName)) {
 137                 log.info("Profile Already Existing - skipping upload")
 138             } else {
 139                 log.info("Uploading K8s Profile..")
 140                 outputPrefixStatuses.put(prefix, OUTPUT_ERROR)
 141                 val profileNamespace: String? = prefixInputParamsMap[INPUT_K8S_PROFILE_NAMESPACE]?.returnNullIfMissing()?.asText()
 142                 var profileSource: String? = prefixInputParamsMap[INPUT_K8S_PROFILE_SOURCE]?.returnNullIfMissing()?.asText()
 143                 if (profileNamespace == null)
 144                     throw BluePrintProcessorException("Profile $profileName namespace is missing")
 145                 if (profileSource == null) {
 146                     profileSource = profileName
 147                     log.info("Profile name used instead of profile source")
 148                 }
 149                 val bluePrintContext = bluePrintRuntimeService.bluePrintContext()
 150                 val artifact: ArtifactDefinition = bluePrintContext.nodeTemplateArtifact(nodeTemplateName, profileSource)
 151                 if (artifact.type != BluePrintConstants.MODEL_TYPE_ARTIFACT_K8S_PROFILE)
 152                     throw BluePrintProcessorException("Unexpected profile artifact type for profile source " +
 153                             "$profileSource. Expecting: $artifact.type")
 154                 var profile = K8sProfile()
 155                 profile.profileName = profileName
 156                 profile.rbName = definitionName
 157                 profile.rbVersion = definitionVersion
 158                 profile.namespace = profileNamespace
 159                 val profileFilePath: Path = prepareProfileFile(profileName, profileSource, artifact.file)
 160                 api.createProfile(profile)
 161                 api.uploadProfileContent(profile, profileFilePath)
 162
 163                 log.info("K8s Profile Upload Completed")
 164                 outputPrefixStatuses.put(prefix, OUTPUT_UPLOADED)
 165             }
 166         }
 167         bluePrintRuntimeService.setNodeTemplateAttributeValue(
 168             nodeTemplateName,
 169             OUTPUT_STATUSES,
 170             outputPrefixStatuses.asJsonNode()
 171         )
 172     }
 173
 174     override suspend fun recoverNB(runtimeException: RuntimeException, executionRequest: ExecutionServiceInput) {
 175         bluePrintRuntimeService.getBluePrintError().addError(runtimeException.message!!)
 176     }
 177
 178     fun getTemplatePrefixList(executionRequest: ExecutionServiceInput): ArrayList<String> {
 179         val result = ArrayList<String>()
 180         for (prefix in executionRequest.payload.get("resource-assignment-request").get("template-prefix").elements())
 181             result.add(prefix.asText())
 182         return result
 183     }
 184
 185     private suspend fun prepareProfileFile(k8sRbProfileName: String, ks8ProfileSource: String, ks8ProfileLocation: String): Path {
 186         val bluePrintContext = bluePrintRuntimeService.bluePrintContext()
 187         val bluePrintBasePath: String = bluePrintContext.rootPath
 188         val profileSourceFileFolderPath: String = bluePrintBasePath.plus(File.separator)
 189                 .plus(ks8ProfileLocation)
 190         val profileFilePathTarGz: String = profileSourceFileFolderPath.plus(".tar.gz")
 191         val profileFilePathTgz: String = profileSourceFileFolderPath.plus(".tgz")
 192
 193         if (Paths.get(profileFilePathTarGz).toFile().exists())
 194             return Paths.get(profileFilePathTarGz)
 195         else if (Paths.get(profileFilePathTgz).toFile().exists())
 196             return Paths.get(profileFilePathTgz)
 197         else if (Paths.get(profileSourceFileFolderPath).toFile().exists()) {
 198             log.info("Profile building started from source $ks8ProfileSource")
 199             val properties: MutableMap<String, Any> = mutableMapOf()
 200             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_STORE_RESULT] = false
 201             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOLUTION_KEY] = ""
 202             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOURCE_ID] = ""
 203             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOURCE_TYPE] = ""
 204             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_OCCURRENCE] = 1
 205             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOLUTION_SUMMARY] = false
 206             val resolutionResult: Pair<String, JsonNode> = resourceResolutionService.resolveResources(
 207                     bluePrintRuntimeService,
 208                     nodeTemplateName,
 209                     ks8ProfileSource,
 210                     properties)
 211             val tempMainPath: File = createTempDir("k8s-profile-", "")
 212             val tempProfilePath: File = createTempDir("content-", "", tempMainPath)
 213
 214             try {
 215                 val manifestFiles: ArrayList<File>? = readManifestFiles(Paths.get(profileSourceFileFolderPath).toFile(),
 216                         tempProfilePath)
 217                 if (manifestFiles != null) {
 218                     templateLocation(Paths.get(profileSourceFileFolderPath).toFile(), resolutionResult.second,
 219                             tempProfilePath, manifestFiles)
 220                 } else
 221                     throw BluePrintProcessorException("Manifest file is missing")
 222                 // Preparation of the final profile content
 223                 val finalProfileFilePath = Paths.get(tempMainPath.toString().plus(File.separator).plus(
 224                         "$k8sRbProfileName.tar.gz"))
 225                 if (!BluePrintArchiveUtils.compress(tempProfilePath, finalProfileFilePath.toFile(),
 226                                 ArchiveType.TarGz)) {
 227                     throw BluePrintProcessorException("Profile compression has failed")
 228                 }
 229                 FileUtils.deleteDirectory(tempProfilePath)
 230
 231                 return finalProfileFilePath
 232             } catch (t: Throwable) {
 233                 FileUtils.deleteDirectory(tempMainPath)
 234                 throw t
 235             }
 236         } else
 237             throw BluePrintProcessorException("Profile source $ks8ProfileSource is missing in CBA folder")
 238     }
 239
 240     private fun readManifestFiles(profileSource: File, destinationFolder: File): ArrayList<File>? {
 241         val directoryListing: Array<File>? = profileSource.listFiles()
 242         var result: ArrayList<File>? = null
 243         if (directoryListing != null) {
 244             for (child in directoryListing) {
 245                 if (!child.isDirectory && child.name.toLowerCase() == "manifest.yaml") {
 246                     child.bufferedReader().use { inr ->
 247                         val manifestYaml = Yaml()
 248                         val manifestObject: Map<String, Any> = manifestYaml.load(inr)
 249                         val typeObject: MutableMap<String, Any>? = manifestObject["type"] as MutableMap<String, Any>?
 250                         if (typeObject != null) {
 251                             result = ArrayList<File>()
 252                             val valuesObject = typeObject["values"]
 253                             if (valuesObject != null) {
 254                                 result!!.add(File(destinationFolder.toString().plus(File.separator).plus(valuesObject)))
 255                                 result!!.add(File(destinationFolder.toString().plus(File.separator).plus(child.name)))
 256                             }
 257                             (typeObject["configresource"] as ArrayList<*>?)?.forEach { item ->
 258                                 val fileInfo: Map<String, Any> = item as Map<String, Any>
 259                                 val filePath = fileInfo["filepath"]
 260                                 val chartPath = fileInfo["chartpath"]
 261                                 if (filePath == null || chartPath == null)
 262                                     log.error("One configresource in manifest was skipped because of the wrong format")
 263                                 else {
 264                                     result!!.add(File(destinationFolder.toString().plus(File.separator).plus(filePath)))
 265                                 }
 266                             }
 267                         }
 268                     }
 269                     break
 270                 }
 271             }
 272         }
 273         return result
 274     }
 275
 276     private fun templateLocation(
 277         location: File,
 278         params: JsonNode,
 279         destinationFolder: File,
 280         manifestFiles: ArrayList<File>
 281     ) {
 282         val directoryListing: Array<File>? = location.listFiles()
 283         if (directoryListing != null) {
 284             for (child in directoryListing) {
 285                 var newDestinationFolder = destinationFolder.toPath()
 286                 if (child.isDirectory)
 287                     newDestinationFolder = Paths.get(destinationFolder.toString().plus(File.separator).plus(child.name))
 288
 289                 templateLocation(child, params, newDestinationFolder.toFile(), manifestFiles)
 290             }
 291         } else if (!location.isDirectory) {
 292             if (location.extension.toLowerCase() == "vtl") {
 293                 templateFile(location, params, destinationFolder, manifestFiles)
 294             } else {
 295                 val finalFilePath = Paths.get(destinationFolder.path.plus(File.separator)
 296                         .plus(location.name)).toFile()
 297                 if (isFileInTheManifestFiles(finalFilePath, manifestFiles)) {
 298                     if (!destinationFolder.exists())
 299                         Files.createDirectories(destinationFolder.toPath())
 300                     FileUtils.copyFile(location, finalFilePath)
 301                 }
 302             }
 303         }
 304     }
 305
 306     private fun isFileInTheManifestFiles(file: File, manifestFiles: ArrayList<File>): Boolean {
 307         manifestFiles.forEach { fileFromManifest ->
 308             if (fileFromManifest.toString().toLowerCase() == file.toString().toLowerCase())
 309                 return true
 310         }
 311         return false
 312     }
 313
 314     private fun templateFile(
 315         templatedFile: File,
 316         params: JsonNode,
 317         destinationFolder: File,
 318         manifestFiles: ArrayList<File>
 319     ) {
 320         val finalFile = File(destinationFolder.path.plus(File.separator)
 321                 .plus(templatedFile.nameWithoutExtension))
 322         if (!isFileInTheManifestFiles(finalFile, manifestFiles))
 323             return
 324         val fileContent = templatedFile.bufferedReader().readText()
 325         val finalFileContent = BluePrintVelocityTemplateService.generateContent(fileContent,
 326                 params, true)
 327         if (!destinationFolder.exists())
 328             Files.createDirectories(destinationFolder.toPath())
 329         finalFile.bufferedWriter().use { out -> out.write(finalFileContent) }
 330     }
 331 }