gerrit.onap Code Review - ccsdk/cds.git/blob

   1 /*
   2  * Copyright © 2017-2018 AT&T Intellectual Property.
   3  * Modifications Copyright © 2019 IBM.
   4  * Modifications Copyright © 2020 Orange.
   5  * Modifications Copyright © 2020 Deutsche Telekom AG.
   6  *
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *     http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  */
  19
  20 package org.onap.ccsdk.cds.blueprintsprocessor.functions.k8s.definition.profile
  21
  22 import com.fasterxml.jackson.databind.JsonNode
  23 import com.fasterxml.jackson.databind.node.ArrayNode
  24 import com.fasterxml.jackson.databind.node.ObjectNode
  25 import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
  26 import com.fasterxml.jackson.module.kotlin.readValue
  27 import org.apache.commons.io.FileUtils
  28 import org.onap.ccsdk.cds.blueprintsprocessor.core.BluePrintPropertiesService
  29 import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.ExecutionServiceInput
  30 import org.onap.ccsdk.cds.blueprintsprocessor.functions.k8s.K8sConnectionPluginConfiguration
  31 import org.onap.ccsdk.cds.blueprintsprocessor.functions.k8s.definition.K8sPluginDefinitionApi
  32 import org.onap.ccsdk.cds.blueprintsprocessor.functions.k8s.instance.K8sRbInstance
  33 import org.onap.ccsdk.cds.blueprintsprocessor.functions.k8s.instance.K8sRbInstanceGvk
  34 import org.onap.ccsdk.cds.controllerblueprints.resource.dict.ResourceAssignment
  35 import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.ResourceResolutionConstants
  36 import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.ResourceResolutionService
  37 import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.AbstractComponentFunction
  38 import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintConstants
  39 import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintProcessorException
  40 import org.onap.ccsdk.cds.controllerblueprints.core.asJsonNode
  41 import org.onap.ccsdk.cds.controllerblueprints.core.data.ArtifactDefinition
  42 import org.onap.ccsdk.cds.controllerblueprints.core.returnNullIfMissing
  43 import org.onap.ccsdk.cds.controllerblueprints.core.service.BluePrintVelocityTemplateService
  44 import org.onap.ccsdk.cds.controllerblueprints.core.utils.ArchiveType
  45 import org.onap.ccsdk.cds.controllerblueprints.core.utils.BluePrintArchiveUtils
  46 import org.onap.ccsdk.cds.controllerblueprints.core.utils.JacksonUtils
  47 import org.slf4j.LoggerFactory
  48 import org.springframework.beans.factory.config.ConfigurableBeanFactory
  49 import org.springframework.context.annotation.Scope
  50 import org.springframework.stereotype.Component
  51 import org.yaml.snakeyaml.Yaml
  52 import java.io.File
  53 import java.nio.file.Files
  54 import java.nio.file.Path
  55 import java.nio.file.Paths
  56
  57 @Component("component-k8s-profile-upload")
  58 @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
  59 open class K8sProfileUploadComponent(
  60     private var bluePrintPropertiesService: BluePrintPropertiesService,
  61     private val resourceResolutionService: ResourceResolutionService
  62 ) :
  63
  64     AbstractComponentFunction() {
  65
  66     companion object {
  67
  68         const val INPUT_K8S_PROFILE_NAME = "k8s-rb-profile-name"
  69         const val INPUT_K8S_DEFINITION_NAME = "k8s-rb-definition-name"
  70         const val INPUT_K8S_DEFINITION_VERSION = "k8s-rb-definition-version"
  71         const val INPUT_K8S_PROFILE_NAMESPACE = "k8s-rb-profile-namespace"
  72         const val INPUT_K8S_PROFILE_LABELS = "k8s-rb-profile-labels"
  73         const val INPUT_K8S_PROFILE_EXTRA_TYPES = "k8s-rb-profile-extra-types"
  74         const val INPUT_K8S_PROFILE_K8S_VERSION = "k8s-rb-profile-k8s-version"
  75         const val INPUT_K8S_PROFILE_SOURCE = "k8s-rb-profile-source"
  76         const val INPUT_RESOURCE_ASSIGNMENT_MAP = "resource-assignment-map"
  77         const val INPUT_ARTIFACT_PREFIX_NAMES = "artifact-prefix-names"
  78
  79         const val OUTPUT_STATUSES = "statuses"
  80         const val OUTPUT_SKIPPED = "skipped"
  81         const val OUTPUT_UPLOADED = "uploaded"
  82         const val OUTPUT_ERROR = "error"
  83     }
  84
  85     private val log = LoggerFactory.getLogger(K8sProfileUploadComponent::class.java)!!
  86
  87     override suspend fun processNB(executionRequest: ExecutionServiceInput) {
  88         log.info("Triggering K8s Profile Upload component logic.")
  89
  90         val inputParameterNames = arrayOf(
  91             INPUT_K8S_PROFILE_NAME,
  92             INPUT_K8S_DEFINITION_NAME,
  93             INPUT_K8S_DEFINITION_VERSION,
  94             INPUT_K8S_PROFILE_NAMESPACE,
  95             INPUT_K8S_PROFILE_LABELS,
  96             INPUT_K8S_PROFILE_EXTRA_TYPES,
  97             INPUT_K8S_PROFILE_K8S_VERSION,
  98             INPUT_K8S_PROFILE_SOURCE,
  99             INPUT_ARTIFACT_PREFIX_NAMES
 100         )
 101         var outputPrefixStatuses = mutableMapOf<String, String>()
 102         var inputParamsMap = mutableMapOf<String, JsonNode?>()
 103
 104         inputParameterNames.forEach {
 105             inputParamsMap[it] = getOptionalOperationInput(it)?.returnNullIfMissing()
 106         }
 107
 108         log.info("Getting the template prefixes")
 109         val prefixList: ArrayList<String> = getTemplatePrefixList(inputParamsMap[INPUT_ARTIFACT_PREFIX_NAMES])
 110
 111         log.info("Iterating over prefixes in resource assignment map.")
 112         for (prefix in prefixList) {
 113             // Prefilling prefix sucess status
 114             outputPrefixStatuses.put(prefix, OUTPUT_SKIPPED)
 115             // Resource assignment map is organized by prefixes, in each iteraton we work only
 116             // on one section of resource assignment map
 117             val prefixNode: JsonNode = operationInputs[INPUT_RESOURCE_ASSIGNMENT_MAP]?.get(prefix) ?: continue
 118             val assignmentMapPrefix = JacksonUtils.jsonNode(prefixNode.toPrettyString()) as ObjectNode
 119
 120             // We are copying the map because for each prefix it might be completed with a different data
 121             var prefixInputParamsMap = inputParamsMap.toMutableMap()
 122             prefixInputParamsMap.forEach { (inputParamName, value) ->
 123                 if (value == null) {
 124                     val mapValue = assignmentMapPrefix?.get(inputParamName)
 125                     log.debug("$inputParamName value was $value so we fetch $mapValue")
 126                     prefixInputParamsMap[inputParamName] = mapValue
 127                 }
 128             }
 129
 130             // For clarity we pull out the required fields
 131             val profileName: String? = prefixInputParamsMap[INPUT_K8S_PROFILE_NAME]?.returnNullIfMissing()?.asText()
 132             val definitionName: String? = prefixInputParamsMap[INPUT_K8S_DEFINITION_NAME]?.returnNullIfMissing()?.asText()
 133             val definitionVersion: String? = prefixInputParamsMap[INPUT_K8S_DEFINITION_VERSION]?.returnNullIfMissing()?.asText()
 134
 135             val k8sProfileUploadConfiguration = K8sConnectionPluginConfiguration(bluePrintPropertiesService)
 136
 137             // Creating API connector
 138             var api = K8sPluginDefinitionApi(k8sProfileUploadConfiguration)
 139
 140             if ((profileName == null) || (definitionName == null) || (definitionVersion == null)) {
 141                 log.warn("Prefix $prefix does not have required data for us to continue.")
 142             } else if (!api.hasDefinition(definitionName, definitionVersion)) {
 143                 throw BluePrintProcessorException("K8s RB Definition ($definitionName/$definitionVersion) not found ")
 144             } else if (profileName == "") {
 145                 log.warn("K8s rb profile name is empty! Either define profile name to use or choose default")
 146             } else if (api.hasProfile(definitionName, definitionVersion, profileName)) {
 147                 log.info("Profile Already Existing - skipping upload")
 148             } else {
 149                 log.info("Uploading K8s Profile..")
 150                 outputPrefixStatuses.put(prefix, OUTPUT_ERROR)
 151                 val profileNamespace: String? = prefixInputParamsMap[INPUT_K8S_PROFILE_NAMESPACE]?.returnNullIfMissing()?.asText()
 152                 val profileK8sVersion: String? = prefixInputParamsMap[INPUT_K8S_PROFILE_K8S_VERSION]?.returnNullIfMissing()?.asText()
 153                 var profileSource: String? = prefixInputParamsMap[INPUT_K8S_PROFILE_SOURCE]?.returnNullIfMissing()?.asText()
 154                 var profileLabels: JsonNode? = prefixInputParamsMap[INPUT_K8S_PROFILE_LABELS]?.returnNullIfMissing()
 155                 var profileExtraTypes: JsonNode? = prefixInputParamsMap[INPUT_K8S_PROFILE_EXTRA_TYPES]?.returnNullIfMissing()
 156                 if (profileNamespace == null)
 157                     throw BluePrintProcessorException("Profile $profileName namespace is missing")
 158                 if (profileSource == null) {
 159                     profileSource = profileName
 160                     log.info("Profile name used instead of profile source")
 161                 }
 162                 val bluePrintContext = bluePrintRuntimeService.bluePrintContext()
 163                 val artifact: ArtifactDefinition = bluePrintContext.nodeTemplateArtifact(nodeTemplateName, profileSource)
 164                 if (artifact.type != BluePrintConstants.MODEL_TYPE_ARTIFACT_K8S_PROFILE)
 165                     throw BluePrintProcessorException(
 166                         "Unexpected profile artifact type for profile source " +
 167                             "$profileSource. Expecting: $artifact.type"
 168                     )
 169                 var profile = K8sProfile()
 170                 profile.profileName = profileName
 171                 profile.rbName = definitionName
 172                 profile.rbVersion = definitionVersion
 173                 profile.namespace = profileNamespace
 174                 if (profileK8sVersion != null)
 175                     profile.kubernetesVersion = profileK8sVersion
 176                 if (profileLabels != null) {
 177                     val objectMapper = jacksonObjectMapper()
 178                     val labelList: HashMap<String, String> = objectMapper.readValue(profileLabels.toPrettyString())
 179                     profile.labels = labelList
 180                 }
 181                 if (profileExtraTypes != null) {
 182                     val objectMapper = jacksonObjectMapper()
 183                     val extraTypeList: ArrayList<K8sRbInstanceGvk> = objectMapper.readValue(profileExtraTypes.toPrettyString())
 184                     profile.extraResourceTypes = extraTypeList
 185                 }
 186                 val profileFilePath: Path = prepareProfileFile(profileName, profileSource, artifact.file)
 187                 api.createProfile(definitionName, definitionVersion, profile)
 188                 api.uploadProfileContent(definitionName, definitionVersion, profile, profileFilePath)
 189
 190                 log.info("K8s Profile Upload Completed")
 191                 outputPrefixStatuses.put(prefix, OUTPUT_UPLOADED)
 192             }
 193         }
 194         bluePrintRuntimeService.setNodeTemplateAttributeValue(
 195             nodeTemplateName,
 196             OUTPUT_STATUSES,
 197             outputPrefixStatuses.asJsonNode()
 198         )
 199     }
 200
 201     override suspend fun recoverNB(runtimeException: RuntimeException, executionRequest: ExecutionServiceInput) {
 202         addError(runtimeException.message!!)
 203     }
 204
 205     private fun getTemplatePrefixList(node: JsonNode?): ArrayList<String> {
 206         var result = ArrayList<String>()
 207         when (node) {
 208             is ArrayNode -> {
 209                 val arrayNode = node.toList()
 210                 for (prefixNode in arrayNode)
 211                     result.add(prefixNode.asText())
 212             }
 213             is ObjectNode -> {
 214                 result.add(node.asText())
 215             }
 216         }
 217         return result
 218     }
 219
 220     private suspend fun prepareProfileFile(k8sRbProfileName: String, ks8ProfileSource: String, ks8ProfileLocation: String): Path {
 221         val bluePrintContext = bluePrintRuntimeService.bluePrintContext()
 222         val bluePrintBasePath: String = bluePrintContext.rootPath
 223         val profileSourceFileFolderPath: Path = Paths.get(
 224             bluePrintBasePath.plus(File.separator).plus(ks8ProfileLocation)
 225         )
 226
 227         if (profileSourceFileFolderPath.toFile().exists() && !profileSourceFileFolderPath.toFile().isDirectory)
 228             return profileSourceFileFolderPath
 229         else if (profileSourceFileFolderPath.toFile().exists()) {
 230             log.info("Profile building started from source $ks8ProfileSource")
 231             val properties: MutableMap<String, Any> = mutableMapOf()
 232             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_STORE_RESULT] = false
 233             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOLUTION_KEY] = ""
 234             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOURCE_ID] = ""
 235             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOURCE_TYPE] = ""
 236             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_OCCURRENCE] = 1
 237             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOLUTION_SUMMARY] = false
 238             val resolutionResult: Pair<String, MutableList<ResourceAssignment>> = resourceResolutionService.resolveResources(
 239                 bluePrintRuntimeService,
 240                 nodeTemplateName,
 241                 ks8ProfileSource,
 242                 properties
 243             )
 244             val tempMainPath: File = createTempDir("k8s-profile-", "")
 245             val tempProfilePath: File = createTempDir("content-", "", tempMainPath)
 246
 247             val resolvedJsonContent = resolutionResult.second
 248                 .associateBy({ it.name }, { it.property?.value })
 249                 .asJsonNode()
 250
 251             try {
 252                 val manifestFiles: ArrayList<File>? = readManifestFiles(
 253                     profileSourceFileFolderPath.toFile(),
 254                     tempProfilePath
 255                 )
 256                 if (manifestFiles != null) {
 257                     templateLocation(
 258                         profileSourceFileFolderPath.toFile(), resolvedJsonContent,
 259                         tempProfilePath, manifestFiles
 260                     )
 261                 } else
 262                     throw BluePrintProcessorException("Manifest file is missing")
 263                 // Preparation of the final profile content
 264                 val finalProfileFilePath = Paths.get(
 265                     tempMainPath.toString().plus(File.separator).plus(
 266                         "$k8sRbProfileName.tar.gz"
 267                     )
 268                 )
 269                 if (!BluePrintArchiveUtils.compress(
 270                         tempProfilePath, finalProfileFilePath.toFile(),
 271                         ArchiveType.TarGz
 272                     )
 273                 ) {
 274                     throw BluePrintProcessorException("Profile compression has failed")
 275                 }
 276                 FileUtils.deleteDirectory(tempProfilePath)
 277
 278                 return finalProfileFilePath
 279             } catch (t: Throwable) {
 280                 FileUtils.deleteDirectory(tempMainPath)
 281                 throw t
 282             }
 283         } else
 284             throw BluePrintProcessorException("Profile source $ks8ProfileLocation is missing in CBA folder")
 285     }
 286
 287     private fun readManifestFiles(profileSource: File, destinationFolder: File): ArrayList<File>? {
 288         val directoryListing: Array<File>? = profileSource.listFiles()
 289         var result: ArrayList<File>? = null
 290         if (directoryListing != null) {
 291             for (child in directoryListing) {
 292                 if (!child.isDirectory && child.name.toLowerCase() == "manifest.yaml") {
 293                     child.bufferedReader().use { inr ->
 294                         val manifestYaml = Yaml()
 295                         val manifestObject: Map<String, Any> = manifestYaml.load(inr)
 296                         val typeObject: MutableMap<String, Any>? = manifestObject["type"] as MutableMap<String, Any>?
 297                         if (typeObject != null) {
 298                             result = ArrayList<File>()
 299                             val valuesObject = typeObject["values"]
 300                             if (valuesObject != null) {
 301                                 result!!.add(File(destinationFolder.toString().plus(File.separator).plus(valuesObject)))
 302                                 result!!.add(File(destinationFolder.toString().plus(File.separator).plus(child.name)))
 303                             }
 304                             (typeObject["configresource"] as ArrayList<*>?)?.forEach { item ->
 305                                 val fileInfo: Map<String, Any> = item as Map<String, Any>
 306                                 val filePath = fileInfo["filepath"]
 307                                 val chartPath = fileInfo["chartpath"]
 308                                 if (filePath == null || chartPath == null)
 309                                     log.error("One configresource in manifest was skipped because of the wrong format")
 310                                 else {
 311                                     result!!.add(File(destinationFolder.toString().plus(File.separator).plus(filePath)))
 312                                 }
 313                             }
 314                         }
 315                     }
 316                     break
 317                 }
 318             }
 319         }
 320         return result
 321     }
 322
 323     private fun templateLocation(
 324         location: File,
 325         params: JsonNode,
 326         destinationFolder: File,
 327         manifestFiles: ArrayList<File>
 328     ) {
 329         val directoryListing: Array<File>? = location.listFiles()
 330         if (directoryListing != null) {
 331             for (child in directoryListing) {
 332                 var newDestinationFolder = destinationFolder.toPath()
 333                 if (child.isDirectory)
 334                     newDestinationFolder = Paths.get(destinationFolder.toString().plus(File.separator).plus(child.name))
 335
 336                 templateLocation(child, params, newDestinationFolder.toFile(), manifestFiles)
 337             }
 338         } else if (!location.isDirectory) {
 339             if (location.extension.toLowerCase() == "vtl") {
 340                 templateFile(location, params, destinationFolder, manifestFiles)
 341             } else {
 342                 val finalFilePath = Paths.get(
 343                     destinationFolder.path.plus(File.separator)
 344                         .plus(location.name)
 345                 ).toFile()
 346                 if (isFileInTheManifestFiles(finalFilePath, manifestFiles)) {
 347                     if (!destinationFolder.exists())
 348                         Files.createDirectories(destinationFolder.toPath())
 349                     FileUtils.copyFile(location, finalFilePath)
 350                 }
 351             }
 352         }
 353     }
 354
 355     private fun isFileInTheManifestFiles(file: File, manifestFiles: ArrayList<File>): Boolean {
 356         manifestFiles.forEach { fileFromManifest ->
 357             if (fileFromManifest.toString().toLowerCase() == file.toString().toLowerCase())
 358                 return true
 359         }
 360         return false
 361     }
 362
 363     private fun templateFile(
 364         templatedFile: File,
 365         params: JsonNode,
 366         destinationFolder: File,
 367         manifestFiles: ArrayList<File>
 368     ) {
 369         val finalFile = File(
 370             destinationFolder.path.plus(File.separator)
 371                 .plus(templatedFile.nameWithoutExtension)
 372         )
 373         if (!isFileInTheManifestFiles(finalFile, manifestFiles))
 374             return
 375         val fileContent = Files.readString(templatedFile.toPath())
 376         val finalFileContent = BluePrintVelocityTemplateService.generateContent(
 377             fileContent,
 378             params.toString(), true
 379         )
 380         if (!destinationFolder.exists())
 381             Files.createDirectories(destinationFolder.toPath())
 382         finalFile.bufferedWriter().use { out -> out.write(finalFileContent) }
 383     }
 384 }