ms/blueprintsprocessor/functions/k8s-connection-plugin/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/k8s/definition/profile/K8sProfileUploadComponent.kt

   1 /*
   2  * Copyright © 2017-2018 AT&T Intellectual Property.
   3  * Modifications Copyright © 2019 IBM.
   4  * Modifications Copyright © 2020 Orange.
   5  * Modifications Copyright © 2020 Deutsche Telekom AG.
   6  *
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *     http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  */
  19
  20 package org.onap.ccsdk.cds.blueprintsprocessor.functions.k8s.definition.profile
  21
  22 import com.fasterxml.jackson.databind.JsonNode
  23 import com.fasterxml.jackson.databind.node.ArrayNode
  24 import com.fasterxml.jackson.databind.node.ObjectNode
  25 import org.apache.commons.io.FileUtils
  26 import org.onap.ccsdk.cds.blueprintsprocessor.core.BlueprintPropertiesService
  27 import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.ExecutionServiceInput
  28 import org.onap.ccsdk.cds.blueprintsprocessor.functions.k8s.K8sConnectionPluginConfiguration
  29 import org.onap.ccsdk.cds.blueprintsprocessor.functions.k8s.definition.K8sPluginDefinitionApi
  30 import org.onap.ccsdk.cds.controllerblueprints.resource.dict.ResourceAssignment
  31 import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.ResourceResolutionConstants
  32 import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.ResourceResolutionService
  33 import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.AbstractComponentFunction
  34 import org.onap.ccsdk.cds.controllerblueprints.core.BlueprintConstants
  35 import org.onap.ccsdk.cds.controllerblueprints.core.BlueprintProcessorException
  36 import org.onap.ccsdk.cds.controllerblueprints.core.asJsonNode
  37 import org.onap.ccsdk.cds.controllerblueprints.core.data.ArtifactDefinition
  38 import org.onap.ccsdk.cds.controllerblueprints.core.returnNullIfMissing
  39 import org.onap.ccsdk.cds.controllerblueprints.core.service.BlueprintVelocityTemplateService
  40 import org.onap.ccsdk.cds.controllerblueprints.core.utils.ArchiveType
  41 import org.onap.ccsdk.cds.controllerblueprints.core.utils.BlueprintArchiveUtils
  42 import org.onap.ccsdk.cds.controllerblueprints.core.utils.JacksonUtils
  43 import org.slf4j.LoggerFactory
  44 import org.springframework.beans.factory.config.ConfigurableBeanFactory
  45 import org.springframework.context.annotation.Scope
  46 import org.springframework.stereotype.Component
  47 import org.yaml.snakeyaml.Yaml
  48 import java.io.File
  49 import java.nio.file.Files
  50 import java.nio.file.Path
  51 import java.nio.file.Paths
  52
  53 @Component("component-k8s-profile-upload")
  54 @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
  55 open class K8sProfileUploadComponent(
  56     private var bluePrintPropertiesService: BlueprintPropertiesService,
  57     private val resourceResolutionService: ResourceResolutionService
  58 ) :
  59
  60     AbstractComponentFunction() {
  61
  62     companion object {
  63
  64         const val INPUT_K8S_PROFILE_NAME = "k8s-rb-profile-name"
  65         const val INPUT_K8S_DEFINITION_NAME = "k8s-rb-definition-name"
  66         const val INPUT_K8S_DEFINITION_VERSION = "k8s-rb-definition-version"
  67         const val INPUT_K8S_PROFILE_NAMESPACE = "k8s-rb-profile-namespace"
  68         const val INPUT_K8S_PROFILE_SOURCE = "k8s-rb-profile-source"
  69         const val INPUT_RESOURCE_ASSIGNMENT_MAP = "resource-assignment-map"
  70         const val INPUT_ARTIFACT_PREFIX_NAMES = "artifact-prefix-names"
  71
  72         const val OUTPUT_STATUSES = "statuses"
  73         const val OUTPUT_SKIPPED = "skipped"
  74         const val OUTPUT_UPLOADED = "uploaded"
  75         const val OUTPUT_ERROR = "error"
  76     }
  77
  78     private val log = LoggerFactory.getLogger(K8sProfileUploadComponent::class.java)!!
  79
  80     override suspend fun processNB(executionRequest: ExecutionServiceInput) {
  81         log.info("Triggering K8s Profile Upload component logic.")
  82
  83         val inputParameterNames = arrayOf(
  84             INPUT_K8S_PROFILE_NAME,
  85             INPUT_K8S_DEFINITION_NAME,
  86             INPUT_K8S_DEFINITION_VERSION,
  87             INPUT_K8S_PROFILE_NAMESPACE,
  88             INPUT_K8S_PROFILE_SOURCE,
  89             INPUT_ARTIFACT_PREFIX_NAMES
  90         )
  91         var outputPrefixStatuses = mutableMapOf<String, String>()
  92         var inputParamsMap = mutableMapOf<String, JsonNode?>()
  93
  94         inputParameterNames.forEach {
  95             inputParamsMap[it] = getOptionalOperationInput(it)?.returnNullIfMissing()
  96         }
  97
  98         log.info("Getting the template prefixes")
  99         val prefixList: ArrayList<String> = getTemplatePrefixList(inputParamsMap[INPUT_ARTIFACT_PREFIX_NAMES])
 100
 101         log.info("Iterating over prefixes in resource assignment map.")
 102         for (prefix in prefixList) {
 103             // Prefilling prefix sucess status
 104             outputPrefixStatuses.put(prefix, OUTPUT_SKIPPED)
 105             // Resource assignment map is organized by prefixes, in each iteraton we work only
 106             // on one section of resource assignment map
 107             val prefixNode: JsonNode = operationInputs[INPUT_RESOURCE_ASSIGNMENT_MAP]?.get(prefix) ?: continue
 108             val assignmentMapPrefix = JacksonUtils.jsonNode(prefixNode.toPrettyString()) as ObjectNode
 109
 110             // We are copying the map because for each prefix it might be completed with a different data
 111             var prefixInputParamsMap = inputParamsMap.toMutableMap()
 112             prefixInputParamsMap.forEach { (inputParamName, value) ->
 113                 if (value == null) {
 114                     val mapValue = assignmentMapPrefix?.get(inputParamName)
 115                     log.debug("$inputParamName value was $value so we fetch $mapValue")
 116                     prefixInputParamsMap[inputParamName] = mapValue
 117                 }
 118             }
 119
 120             // For clarity we pull out the required fields
 121             val profileName: String? = prefixInputParamsMap[INPUT_K8S_PROFILE_NAME]?.returnNullIfMissing()?.asText()
 122             val definitionName: String? = prefixInputParamsMap[INPUT_K8S_DEFINITION_NAME]?.returnNullIfMissing()?.asText()
 123             val definitionVersion: String? = prefixInputParamsMap[INPUT_K8S_DEFINITION_VERSION]?.returnNullIfMissing()?.asText()
 124
 125             val k8sProfileUploadConfiguration = K8sConnectionPluginConfiguration(bluePrintPropertiesService)
 126
 127             // Creating API connector
 128             var api = K8sPluginDefinitionApi(k8sProfileUploadConfiguration)
 129
 130             if ((profileName == null) || (definitionName == null) || (definitionVersion == null)) {
 131                 log.warn("Prefix $prefix does not have required data for us to continue.")
 132             } else if (!api.hasDefinition(definitionName, definitionVersion)) {
 133                 log.warn("K8s RB Definition ($definitionName/$definitionVersion) not found ")
 134             } else if (profileName == "") {
 135                 log.warn("K8s rb profile name is empty! Either define profile name to use or choose default")
 136             } else if (api.hasProfile(definitionName, definitionVersion, profileName)) {
 137                 log.info("Profile Already Existing - skipping upload")
 138             } else {
 139                 log.info("Uploading K8s Profile..")
 140                 outputPrefixStatuses.put(prefix, OUTPUT_ERROR)
 141                 val profileNamespace: String? = prefixInputParamsMap[INPUT_K8S_PROFILE_NAMESPACE]?.returnNullIfMissing()?.asText()
 142                 var profileSource: String? = prefixInputParamsMap[INPUT_K8S_PROFILE_SOURCE]?.returnNullIfMissing()?.asText()
 143                 if (profileNamespace == null)
 144                     throw BlueprintProcessorException("Profile $profileName namespace is missing")
 145                 if (profileSource == null) {
 146                     profileSource = profileName
 147                     log.info("Profile name used instead of profile source")
 148                 }
 149                 val bluePrintContext = bluePrintRuntimeService.bluePrintContext()
 150                 val artifact: ArtifactDefinition = bluePrintContext.nodeTemplateArtifact(nodeTemplateName, profileSource)
 151                 if (artifact.type != BlueprintConstants.MODEL_TYPE_ARTIFACT_K8S_PROFILE)
 152                     throw BlueprintProcessorException(
 153                         "Unexpected profile artifact type for profile source " +
 154                             "$profileSource. Expecting: $artifact.type"
 155                     )
 156                 var profile = K8sProfile()
 157                 profile.profileName = profileName
 158                 profile.rbName = definitionName
 159                 profile.rbVersion = definitionVersion
 160                 profile.namespace = profileNamespace
 161                 val profileFilePath: Path = prepareProfileFile(profileName, profileSource, artifact.file)
 162                 api.createProfile(definitionName, definitionVersion, profile)
 163                 api.uploadProfileContent(definitionName, definitionVersion, profile, profileFilePath)
 164
 165                 log.info("K8s Profile Upload Completed")
 166                 outputPrefixStatuses.put(prefix, OUTPUT_UPLOADED)
 167             }
 168         }
 169         bluePrintRuntimeService.setNodeTemplateAttributeValue(
 170             nodeTemplateName,
 171             OUTPUT_STATUSES,
 172             outputPrefixStatuses.asJsonNode()
 173         )
 174     }
 175
 176     override suspend fun recoverNB(runtimeException: RuntimeException, executionRequest: ExecutionServiceInput) {
 177         bluePrintRuntimeService.getBlueprintError().addError(runtimeException.message!!)
 178     }
 179
 180     private fun getTemplatePrefixList(node: JsonNode?): ArrayList<String> {
 181         var result = ArrayList<String>()
 182         when (node) {
 183             is ArrayNode -> {
 184                 val arrayNode = node.toList()
 185                 for (prefixNode in arrayNode)
 186                     result.add(prefixNode.asText())
 187             }
 188             is ObjectNode -> {
 189                 result.add(node.asText())
 190             }
 191         }
 192         return result
 193     }
 194
 195     private suspend fun prepareProfileFile(k8sRbProfileName: String, ks8ProfileSource: String, ks8ProfileLocation: String): Path {
 196         val bluePrintContext = bluePrintRuntimeService.bluePrintContext()
 197         val bluePrintBasePath: String = bluePrintContext.rootPath
 198         val profileSourceFileFolderPath: Path = Paths.get(
 199             bluePrintBasePath.plus(File.separator).plus(ks8ProfileLocation)
 200         )
 201
 202         if (profileSourceFileFolderPath.toFile().exists() && !profileSourceFileFolderPath.toFile().isDirectory)
 203             return profileSourceFileFolderPath
 204         else if (profileSourceFileFolderPath.toFile().exists()) {
 205             log.info("Profile building started from source $ks8ProfileSource")
 206             val properties: MutableMap<String, Any> = mutableMapOf()
 207             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_STORE_RESULT] = false
 208             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOLUTION_KEY] = ""
 209             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOURCE_ID] = ""
 210             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOURCE_TYPE] = ""
 211             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_OCCURRENCE] = 1
 212             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOLUTION_SUMMARY] = false
 213             val resolutionResult: Pair<String, MutableList<ResourceAssignment>> = resourceResolutionService.resolveResources(
 214                 bluePrintRuntimeService,
 215                 nodeTemplateName,
 216                 ks8ProfileSource,
 217                 properties
 218             )
 219             val tempMainPath: File = createTempDir("k8s-profile-", "")
 220             val tempProfilePath: File = createTempDir("content-", "", tempMainPath)
 221
 222             val resolvedJsonContent = resolutionResult.second
 223                 .associateBy({ it.name }, { it.property?.value })
 224                 .asJsonNode()
 225
 226             try {
 227                 val manifestFiles: ArrayList<File>? = readManifestFiles(
 228                     profileSourceFileFolderPath.toFile(),
 229                     tempProfilePath
 230                 )
 231                 if (manifestFiles != null) {
 232                     templateLocation(
 233                         profileSourceFileFolderPath.toFile(), resolvedJsonContent,
 234                         tempProfilePath, manifestFiles
 235                     )
 236                 } else
 237                     throw BlueprintProcessorException("Manifest file is missing")
 238                 // Preparation of the final profile content
 239                 val finalProfileFilePath = Paths.get(
 240                     tempMainPath.toString().plus(File.separator).plus(
 241                         "$k8sRbProfileName.tar.gz"
 242                     )
 243                 )
 244                 if (!BlueprintArchiveUtils.compress(
 245                         tempProfilePath, finalProfileFilePath.toFile(),
 246                         ArchiveType.TarGz
 247                     )
 248                 ) {
 249                     throw BlueprintProcessorException("Profile compression has failed")
 250                 }
 251                 FileUtils.deleteDirectory(tempProfilePath)
 252
 253                 return finalProfileFilePath
 254             } catch (t: Throwable) {
 255                 FileUtils.deleteDirectory(tempMainPath)
 256                 throw t
 257             }
 258         } else
 259             throw BlueprintProcessorException("Profile source $ks8ProfileLocation is missing in CBA folder")
 260     }
 261
 262     private fun readManifestFiles(profileSource: File, destinationFolder: File): ArrayList<File>? {
 263         val directoryListing: Array<File>? = profileSource.listFiles()
 264         var result: ArrayList<File>? = null
 265         if (directoryListing != null) {
 266             for (child in directoryListing) {
 267                 if (!child.isDirectory && child.name.toLowerCase() == "manifest.yaml") {
 268                     child.bufferedReader().use { inr ->
 269                         val manifestYaml = Yaml()
 270                         val manifestObject: Map<String, Any> = manifestYaml.load(inr)
 271                         val typeObject: MutableMap<String, Any>? = manifestObject["type"] as MutableMap<String, Any>?
 272                         if (typeObject != null) {
 273                             result = ArrayList<File>()
 274                             val valuesObject = typeObject["values"]
 275                             if (valuesObject != null) {
 276                                 result!!.add(File(destinationFolder.toString().plus(File.separator).plus(valuesObject)))
 277                                 result!!.add(File(destinationFolder.toString().plus(File.separator).plus(child.name)))
 278                             }
 279                             (typeObject["configresource"] as ArrayList<*>?)?.forEach { item ->
 280                                 val fileInfo: Map<String, Any> = item as Map<String, Any>
 281                                 val filePath = fileInfo["filepath"]
 282                                 val chartPath = fileInfo["chartpath"]
 283                                 if (filePath == null || chartPath == null)
 284                                     log.error("One configresource in manifest was skipped because of the wrong format")
 285                                 else {
 286                                     result!!.add(File(destinationFolder.toString().plus(File.separator).plus(filePath)))
 287                                 }
 288                             }
 289                         }
 290                     }
 291                     break
 292                 }
 293             }
 294         }
 295         return result
 296     }
 297
 298     private fun templateLocation(
 299         location: File,
 300         params: JsonNode,
 301         destinationFolder: File,
 302         manifestFiles: ArrayList<File>
 303     ) {
 304         val directoryListing: Array<File>? = location.listFiles()
 305         if (directoryListing != null) {
 306             for (child in directoryListing) {
 307                 var newDestinationFolder = destinationFolder.toPath()
 308                 if (child.isDirectory)
 309                     newDestinationFolder = Paths.get(destinationFolder.toString().plus(File.separator).plus(child.name))
 310
 311                 templateLocation(child, params, newDestinationFolder.toFile(), manifestFiles)
 312             }
 313         } else if (!location.isDirectory) {
 314             if (location.extension.toLowerCase() == "vtl") {
 315                 templateFile(location, params, destinationFolder, manifestFiles)
 316             } else {
 317                 val finalFilePath = Paths.get(
 318                     destinationFolder.path.plus(File.separator)
 319                         .plus(location.name)
 320                 ).toFile()
 321                 if (isFileInTheManifestFiles(finalFilePath, manifestFiles)) {
 322                     if (!destinationFolder.exists())
 323                         Files.createDirectories(destinationFolder.toPath())
 324                     FileUtils.copyFile(location, finalFilePath)
 325                 }
 326             }
 327         }
 328     }
 329
 330     private fun isFileInTheManifestFiles(file: File, manifestFiles: ArrayList<File>): Boolean {
 331         manifestFiles.forEach { fileFromManifest ->
 332             if (fileFromManifest.toString().toLowerCase() == file.toString().toLowerCase())
 333                 return true
 334         }
 335         return false
 336     }
 337
 338     private fun templateFile(
 339         templatedFile: File,
 340         params: JsonNode,
 341         destinationFolder: File,
 342         manifestFiles: ArrayList<File>
 343     ) {
 344         val finalFile = File(
 345             destinationFolder.path.plus(File.separator)
 346                 .plus(templatedFile.nameWithoutExtension)
 347         )
 348         if (!isFileInTheManifestFiles(finalFile, manifestFiles))
 349             return
 350         val fileContent = templatedFile.bufferedReader().readText()
 351         val finalFileContent = BlueprintVelocityTemplateService.generateContent(
 352             fileContent,
 353             params, true
 354         )
 355         if (!destinationFolder.exists())
 356             Files.createDirectories(destinationFolder.toPath())
 357         finalFile.bufferedWriter().use { out -> out.write(finalFileContent) }
 358     }
 359 }