gerrit.onap Code Review - ccsdk/cds.git/blob

   1 /*
   2  * Copyright © 2017-2018 AT&T Intellectual Property.
   3  * Modifications Copyright © 2019 IBM.
   4  * Modifications Copyright © 2020 Orange.
   5  * Modifications Copyright © 2020 Deutsche Telekom AG.
   6  *
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *     http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  */
  19
  20 package org.onap.ccsdk.cds.blueprintsprocessor.functions.k8s.profile.upload
  21
  22 import com.fasterxml.jackson.databind.JsonNode
  23 import com.fasterxml.jackson.databind.node.ArrayNode
  24 import com.fasterxml.jackson.databind.node.ObjectNode
  25 import org.apache.commons.io.FileUtils
  26 import org.onap.ccsdk.cds.blueprintsprocessor.core.BluePrintPropertiesService
  27 import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.ExecutionServiceInput
  28 import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.ResourceResolutionConstants
  29 import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.ResourceResolutionService
  30 import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.AbstractComponentFunction
  31 import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintConstants
  32 import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintProcessorException
  33 import org.onap.ccsdk.cds.controllerblueprints.core.asJsonNode
  34 import org.onap.ccsdk.cds.controllerblueprints.core.data.ArtifactDefinition
  35 import org.onap.ccsdk.cds.controllerblueprints.core.returnNullIfMissing
  36 import org.onap.ccsdk.cds.controllerblueprints.core.service.BluePrintVelocityTemplateService
  37 import org.onap.ccsdk.cds.controllerblueprints.core.utils.ArchiveType
  38 import org.onap.ccsdk.cds.controllerblueprints.core.utils.BluePrintArchiveUtils
  39 import org.onap.ccsdk.cds.controllerblueprints.core.utils.JacksonUtils
  40 import org.slf4j.LoggerFactory
  41 import org.springframework.beans.factory.config.ConfigurableBeanFactory
  42 import org.springframework.context.annotation.Scope
  43 import org.springframework.stereotype.Component
  44 import org.yaml.snakeyaml.Yaml
  45 import java.io.File
  46 import java.nio.file.Files
  47 import java.nio.file.Path
  48 import java.nio.file.Paths
  49 import kotlin.collections.ArrayList
  50
  51 @Component("component-k8s-profile-upload")
  52 @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
  53 open class K8sProfileUploadComponent(
  54     private var bluePrintPropertiesService: BluePrintPropertiesService,
  55     private val resourceResolutionService: ResourceResolutionService
  56 ) :
  57
  58     AbstractComponentFunction() {
  59
  60     companion object {
  61         const val INPUT_K8S_PROFILE_NAME = "k8s-rb-profile-name"
  62         const val INPUT_K8S_DEFINITION_NAME = "k8s-rb-definition-name"
  63         const val INPUT_K8S_DEFINITION_VERSION = "k8s-rb-definition-version"
  64         const val INPUT_K8S_PROFILE_NAMESPACE = "k8s-rb-profile-namespace"
  65         const val INPUT_K8S_PROFILE_SOURCE = "k8s-rb-profile-source"
  66         const val INPUT_RESOURCE_ASSIGNMENT_MAP = "resource-assignment-map"
  67         const val INPUT_ARTIFACT_PREFIX_NAMES = "artifact-prefix-names"
  68
  69         const val OUTPUT_STATUSES = "statuses"
  70         const val OUTPUT_SKIPPED = "skipped"
  71         const val OUTPUT_UPLOADED = "uploaded"
  72         const val OUTPUT_ERROR = "error"
  73     }
  74
  75     private val log = LoggerFactory.getLogger(K8sProfileUploadComponent::class.java)!!
  76
  77     override suspend fun processNB(executionRequest: ExecutionServiceInput) {
  78         log.info("Triggering K8s Profile Upload component logic.")
  79
  80         val inputParameterNames = arrayOf(
  81             INPUT_K8S_PROFILE_NAME,
  82             INPUT_K8S_DEFINITION_NAME,
  83             INPUT_K8S_DEFINITION_VERSION,
  84             INPUT_K8S_PROFILE_NAMESPACE,
  85             INPUT_K8S_PROFILE_SOURCE,
  86             INPUT_ARTIFACT_PREFIX_NAMES
  87         )
  88         var outputPrefixStatuses = mutableMapOf<String, String>()
  89         var inputParamsMap = mutableMapOf<String, JsonNode?>()
  90
  91         inputParameterNames.forEach {
  92             inputParamsMap[it] = getOptionalOperationInput(it)?.returnNullIfMissing()
  93         }
  94
  95         log.info("Getting the template prefixes")
  96         val prefixList: ArrayList<String> = getTemplatePrefixList(inputParamsMap[INPUT_ARTIFACT_PREFIX_NAMES])
  97
  98         log.info("Iterating over prefixes in resource assignment map.")
  99         for (prefix in prefixList) {
 100             // Prefilling prefix sucess status
 101             outputPrefixStatuses.put(prefix, OUTPUT_SKIPPED)
 102             // Resource assignment map is organized by prefixes, in each iteraton we work only
 103             // on one section of resource assignment map
 104             val prefixNode: JsonNode = operationInputs[INPUT_RESOURCE_ASSIGNMENT_MAP]?.get(prefix) ?: continue
 105             val assignmentMapPrefix = JacksonUtils.jsonNode(prefixNode.toPrettyString()) as ObjectNode
 106
 107             // We are copying the map because for each prefix it might be completed with a different data
 108             var prefixInputParamsMap = inputParamsMap.toMutableMap()
 109             prefixInputParamsMap.forEach { (inputParamName, value) ->
 110                 if (value == null) {
 111                     val mapValue = assignmentMapPrefix?.get(inputParamName)
 112                     log.debug("$inputParamName value was $value so we fetch $mapValue")
 113                     prefixInputParamsMap[inputParamName] = mapValue
 114                 }
 115             }
 116
 117             // For clarity we pull out the required fields
 118             val profileName: String? = prefixInputParamsMap[INPUT_K8S_PROFILE_NAME]?.returnNullIfMissing()?.asText()
 119             val definitionName: String? = prefixInputParamsMap[INPUT_K8S_DEFINITION_NAME]?.returnNullIfMissing()?.asText()
 120             val definitionVersion: String? = prefixInputParamsMap[INPUT_K8S_DEFINITION_VERSION]?.returnNullIfMissing()?.asText()
 121
 122             val k8sProfileUploadConfiguration = K8sProfileUploadConfiguration(bluePrintPropertiesService)
 123
 124             // Creating API connector
 125             var api = K8sPluginApi(
 126                 k8sProfileUploadConfiguration.getProperties().username,
 127                 k8sProfileUploadConfiguration.getProperties().password,
 128                 k8sProfileUploadConfiguration.getProperties().url,
 129                 definitionName,
 130                 definitionVersion
 131             )
 132
 133             if ((profileName == null) || (definitionName == null) || (definitionVersion == null)) {
 134                 log.warn("Prefix $prefix does not have required data for us to continue.")
 135             } else if (!api.hasDefinition()) {
 136                 log.warn("K8s RB Definition ($definitionName/$definitionVersion) not found ")
 137             } else if (profileName == "") {
 138                 log.warn("K8s rb profile name is empty! Either define profile name to use or choose default")
 139             } else if (api.hasProfile(profileName)) {
 140                 log.info("Profile Already Existing - skipping upload")
 141             } else {
 142                 log.info("Uploading K8s Profile..")
 143                 outputPrefixStatuses.put(prefix, OUTPUT_ERROR)
 144                 val profileNamespace: String? = prefixInputParamsMap[INPUT_K8S_PROFILE_NAMESPACE]?.returnNullIfMissing()?.asText()
 145                 var profileSource: String? = prefixInputParamsMap[INPUT_K8S_PROFILE_SOURCE]?.returnNullIfMissing()?.asText()
 146                 if (profileNamespace == null)
 147                     throw BluePrintProcessorException("Profile $profileName namespace is missing")
 148                 if (profileSource == null) {
 149                     profileSource = profileName
 150                     log.info("Profile name used instead of profile source")
 151                 }
 152                 val bluePrintContext = bluePrintRuntimeService.bluePrintContext()
 153                 val artifact: ArtifactDefinition = bluePrintContext.nodeTemplateArtifact(nodeTemplateName, profileSource)
 154                 if (artifact.type != BluePrintConstants.MODEL_TYPE_ARTIFACT_K8S_PROFILE)
 155                     throw BluePrintProcessorException("Unexpected profile artifact type for profile source " +
 156                             "$profileSource. Expecting: $artifact.type")
 157                 var profile = K8sProfile()
 158                 profile.profileName = profileName
 159                 profile.rbName = definitionName
 160                 profile.rbVersion = definitionVersion
 161                 profile.namespace = profileNamespace
 162                 val profileFilePath: Path = prepareProfileFile(profileName, profileSource, artifact.file)
 163                 api.createProfile(profile)
 164                 api.uploadProfileContent(profile, profileFilePath)
 165
 166                 log.info("K8s Profile Upload Completed")
 167                 outputPrefixStatuses.put(prefix, OUTPUT_UPLOADED)
 168             }
 169         }
 170         bluePrintRuntimeService.setNodeTemplateAttributeValue(
 171             nodeTemplateName,
 172             OUTPUT_STATUSES,
 173             outputPrefixStatuses.asJsonNode()
 174         )
 175     }
 176
 177     override suspend fun recoverNB(runtimeException: RuntimeException, executionRequest: ExecutionServiceInput) {
 178         bluePrintRuntimeService.getBluePrintError().addError(runtimeException.message!!)
 179     }
 180
 181     private fun getTemplatePrefixList(node: JsonNode?): ArrayList<String> {
 182         var result = ArrayList<String>()
 183         when (node) {
 184             is ArrayNode -> {
 185                 val arrayNode = node.toList()
 186                 for (prefixNode in arrayNode)
 187                     result.add(prefixNode.asText())
 188             }
 189             is ObjectNode -> {
 190                 result.add(node.asText())
 191             }
 192         }
 193         return result
 194     }
 195
 196     private suspend fun prepareProfileFile(k8sRbProfileName: String, ks8ProfileSource: String, ks8ProfileLocation: String): Path {
 197         val bluePrintContext = bluePrintRuntimeService.bluePrintContext()
 198         val bluePrintBasePath: String = bluePrintContext.rootPath
 199         val profileSourceFileFolderPath: String = bluePrintBasePath.plus(File.separator)
 200                 .plus(ks8ProfileLocation)
 201         val profileFilePathTarGz: String = profileSourceFileFolderPath.plus(".tar.gz")
 202         val profileFilePathTgz: String = profileSourceFileFolderPath.plus(".tgz")
 203
 204         if (Paths.get(profileFilePathTarGz).toFile().exists())
 205             return Paths.get(profileFilePathTarGz)
 206         else if (Paths.get(profileFilePathTgz).toFile().exists())
 207             return Paths.get(profileFilePathTgz)
 208         else if (Paths.get(profileSourceFileFolderPath).toFile().exists()) {
 209             log.info("Profile building started from source $ks8ProfileSource")
 210             val properties: MutableMap<String, Any> = mutableMapOf()
 211             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_STORE_RESULT] = false
 212             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOLUTION_KEY] = ""
 213             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOURCE_ID] = ""
 214             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOURCE_TYPE] = ""
 215             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_OCCURRENCE] = 1
 216             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOLUTION_SUMMARY] = false
 217             val resolutionResult: Pair<String, JsonNode> = resourceResolutionService.resolveResources(
 218                     bluePrintRuntimeService,
 219                     nodeTemplateName,
 220                     ks8ProfileSource,
 221                     properties)
 222             val tempMainPath: File = createTempDir("k8s-profile-", "")
 223             val tempProfilePath: File = createTempDir("content-", "", tempMainPath)
 224
 225             try {
 226                 val manifestFiles: ArrayList<File>? = readManifestFiles(Paths.get(profileSourceFileFolderPath).toFile(),
 227                         tempProfilePath)
 228                 if (manifestFiles != null) {
 229                     templateLocation(Paths.get(profileSourceFileFolderPath).toFile(), resolutionResult.second,
 230                             tempProfilePath, manifestFiles)
 231                 } else
 232                     throw BluePrintProcessorException("Manifest file is missing")
 233                 // Preparation of the final profile content
 234                 val finalProfileFilePath = Paths.get(tempMainPath.toString().plus(File.separator).plus(
 235                         "$k8sRbProfileName.tar.gz"))
 236                 if (!BluePrintArchiveUtils.compress(tempProfilePath, finalProfileFilePath.toFile(),
 237                                 ArchiveType.TarGz)) {
 238                     throw BluePrintProcessorException("Profile compression has failed")
 239                 }
 240                 FileUtils.deleteDirectory(tempProfilePath)
 241
 242                 return finalProfileFilePath
 243             } catch (t: Throwable) {
 244                 FileUtils.deleteDirectory(tempMainPath)
 245                 throw t
 246             }
 247         } else
 248             throw BluePrintProcessorException("Profile source $ks8ProfileSource is missing in CBA folder")
 249     }
 250
 251     private fun readManifestFiles(profileSource: File, destinationFolder: File): ArrayList<File>? {
 252         val directoryListing: Array<File>? = profileSource.listFiles()
 253         var result: ArrayList<File>? = null
 254         if (directoryListing != null) {
 255             for (child in directoryListing) {
 256                 if (!child.isDirectory && child.name.toLowerCase() == "manifest.yaml") {
 257                     child.bufferedReader().use { inr ->
 258                         val manifestYaml = Yaml()
 259                         val manifestObject: Map<String, Any> = manifestYaml.load(inr)
 260                         val typeObject: MutableMap<String, Any>? = manifestObject["type"] as MutableMap<String, Any>?
 261                         if (typeObject != null) {
 262                             result = ArrayList<File>()
 263                             val valuesObject = typeObject["values"]
 264                             if (valuesObject != null) {
 265                                 result!!.add(File(destinationFolder.toString().plus(File.separator).plus(valuesObject)))
 266                                 result!!.add(File(destinationFolder.toString().plus(File.separator).plus(child.name)))
 267                             }
 268                             (typeObject["configresource"] as ArrayList<*>?)?.forEach { item ->
 269                                 val fileInfo: Map<String, Any> = item as Map<String, Any>
 270                                 val filePath = fileInfo["filepath"]
 271                                 val chartPath = fileInfo["chartpath"]
 272                                 if (filePath == null || chartPath == null)
 273                                     log.error("One configresource in manifest was skipped because of the wrong format")
 274                                 else {
 275                                     result!!.add(File(destinationFolder.toString().plus(File.separator).plus(filePath)))
 276                                 }
 277                             }
 278                         }
 279                     }
 280                     break
 281                 }
 282             }
 283         }
 284         return result
 285     }
 286
 287     private fun templateLocation(
 288         location: File,
 289         params: JsonNode,
 290         destinationFolder: File,
 291         manifestFiles: ArrayList<File>
 292     ) {
 293         val directoryListing: Array<File>? = location.listFiles()
 294         if (directoryListing != null) {
 295             for (child in directoryListing) {
 296                 var newDestinationFolder = destinationFolder.toPath()
 297                 if (child.isDirectory)
 298                     newDestinationFolder = Paths.get(destinationFolder.toString().plus(File.separator).plus(child.name))
 299
 300                 templateLocation(child, params, newDestinationFolder.toFile(), manifestFiles)
 301             }
 302         } else if (!location.isDirectory) {
 303             if (location.extension.toLowerCase() == "vtl") {
 304                 templateFile(location, params, destinationFolder, manifestFiles)
 305             } else {
 306                 val finalFilePath = Paths.get(destinationFolder.path.plus(File.separator)
 307                         .plus(location.name)).toFile()
 308                 if (isFileInTheManifestFiles(finalFilePath, manifestFiles)) {
 309                     if (!destinationFolder.exists())
 310                         Files.createDirectories(destinationFolder.toPath())
 311                     FileUtils.copyFile(location, finalFilePath)
 312                 }
 313             }
 314         }
 315     }
 316
 317     private fun isFileInTheManifestFiles(file: File, manifestFiles: ArrayList<File>): Boolean {
 318         manifestFiles.forEach { fileFromManifest ->
 319             if (fileFromManifest.toString().toLowerCase() == file.toString().toLowerCase())
 320                 return true
 321         }
 322         return false
 323     }
 324
 325     private fun templateFile(
 326         templatedFile: File,
 327         params: JsonNode,
 328         destinationFolder: File,
 329         manifestFiles: ArrayList<File>
 330     ) {
 331         val finalFile = File(destinationFolder.path.plus(File.separator)
 332                 .plus(templatedFile.nameWithoutExtension))
 333         if (!isFileInTheManifestFiles(finalFile, manifestFiles))
 334             return
 335         val fileContent = templatedFile.bufferedReader().readText()
 336         val finalFileContent = BluePrintVelocityTemplateService.generateContent(fileContent,
 337                 params, true)
 338         if (!destinationFolder.exists())
 339             Files.createDirectories(destinationFolder.toPath())
 340         finalFile.bufferedWriter().use { out -> out.write(finalFileContent) }
 341     }
 342 }