gerrit.onap Code Review - ccsdk/cds.git/blob

   1 /*
   2  *  Copyright © 2017-2018 AT&T Intellectual Property.
   3  *  Modifications Copyright © 2018-2019 IBM, Bell Canada
   4  *
   5  *  Licensed under the Apache License, Version 2.0 (the "License");
   6  *  you may not use this file except in compliance with the License.
   7  *  You may obtain a copy of the License at
   8  *
   9  *      http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  *  Unless required by applicable law or agreed to in writing, software
  12  *  distributed under the License is distributed on an "AS IS" BASIS,
  13  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  *  See the License for the specific language governing permissions and
  15  *  limitations under the License.
  16  */
  17
  18 package org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution
  19
  20 import com.fasterxml.jackson.databind.JsonNode
  21 import kotlinx.coroutines.async
  22 import kotlinx.coroutines.awaitAll
  23 import kotlinx.coroutines.coroutineScope
  24 import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.db.ResourceResolution
  25 import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.db.ResourceResolutionDBService
  26 import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.db.TemplateResolutionService
  27 import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.processor.ResourceAssignmentProcessor
  28 import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.utils.ResourceAssignmentUtils
  29 import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.utils.ResourceDefinitionUtils.createResourceAssignments
  30 import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintConstants
  31 import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintProcessorException
  32 import org.onap.ccsdk.cds.controllerblueprints.core.asJsonNode
  33 import org.onap.ccsdk.cds.controllerblueprints.core.asJsonPrimitive
  34 import org.onap.ccsdk.cds.controllerblueprints.core.asJsonType
  35 import org.onap.ccsdk.cds.controllerblueprints.core.checkNotEmpty
  36 import org.onap.ccsdk.cds.controllerblueprints.core.common.ApplicationConstants.LOG_REDACTED
  37 import org.onap.ccsdk.cds.controllerblueprints.core.service.BluePrintRuntimeService
  38 import org.onap.ccsdk.cds.controllerblueprints.core.service.BluePrintTemplateService
  39 import org.onap.ccsdk.cds.controllerblueprints.core.utils.JacksonUtils
  40 import org.onap.ccsdk.cds.controllerblueprints.core.utils.PropertyDefinitionUtils.Companion.hasLogProtect
  41 import org.onap.ccsdk.cds.controllerblueprints.resource.dict.ResourceAssignment
  42 import org.onap.ccsdk.cds.controllerblueprints.resource.dict.ResourceDefinition
  43 import org.onap.ccsdk.cds.controllerblueprints.resource.dict.utils.BulkResourceSequencingUtils
  44 import org.slf4j.LoggerFactory
  45 import org.springframework.context.ApplicationContext
  46 import org.springframework.stereotype.Service
  47 import java.util.UUID
  48
  49 data class ResourceResolutionResult(
  50     val templateMap: MutableMap<String, String>,
  51     val assignmentMap: MutableMap<String, JsonNode>
  52 )
  53
  54 interface ResourceResolutionService {
  55
  56     fun registeredResourceSources(): List<String>
  57
  58     suspend fun resolveFromDatabase(
  59         bluePrintRuntimeService: BluePrintRuntimeService<*>,
  60         artifactTemplate: String,
  61         resolutionKey: String
  62     ): String
  63
  64     suspend fun resolveResolutionKeysFromDatabase(
  65         bluePrintRuntimeService: BluePrintRuntimeService<*>,
  66         artifactTemplate: String
  67     ): List<String>
  68
  69     suspend fun resolveArtifactNamesAndResolutionKeysFromDatabase(
  70         bluePrintRuntimeService: BluePrintRuntimeService<*>
  71     ): Map<String, List<String>>
  72
  73     suspend fun resolveResources(
  74         bluePrintRuntimeService: BluePrintRuntimeService<*>,
  75         nodeTemplateName: String,
  76         artifactNames: List<String>,
  77         properties: Map<String, Any>,
  78         stepName: String
  79     ): ResourceResolutionResult
  80
  81     suspend fun resolveResources(
  82         bluePrintRuntimeService: BluePrintRuntimeService<*>,
  83         nodeTemplateName: String,
  84         artifactPrefix: String,
  85         properties: Map<String, Any>
  86     ): Pair<String, MutableList<ResourceAssignment>>
  87
  88     /** Resolve resources for all the sources defined in a particular resource Definition[resolveDefinition]
  89      * with other [resourceDefinitions] dependencies for the sources [sources]
  90      * Used to get the same resource values from multiple sources. **/
  91     suspend fun resolveResourceDefinition(
  92         blueprintRuntimeService: BluePrintRuntimeService<*>,
  93         resourceDefinitions: MutableMap<String, ResourceDefinition>,
  94         resolveDefinition: String,
  95         sources: List<String>
  96     ):
  97         MutableMap<String, JsonNode>
  98
  99     suspend fun resolveResourceAssignments(
 100         blueprintRuntimeService: BluePrintRuntimeService<*>,
 101         resourceDefinitions: MutableMap<String, ResourceDefinition>,
 102         resourceAssignments: MutableList<ResourceAssignment>,
 103         artifactPrefix: String,
 104         properties: Map<String, Any>
 105     )
 106 }
 107
 108 @Service(ResourceResolutionConstants.SERVICE_RESOURCE_RESOLUTION)
 109 open class ResourceResolutionServiceImpl(
 110     private var applicationContext: ApplicationContext,
 111     private var templateResolutionDBService: TemplateResolutionService,
 112     private var blueprintTemplateService: BluePrintTemplateService,
 113     private var resourceResolutionDBService: ResourceResolutionDBService
 114 ) :
 115     ResourceResolutionService {
 116
 117     private val log = LoggerFactory.getLogger(ResourceResolutionService::class.java)
 118
 119     override fun registeredResourceSources(): List<String> {
 120         return applicationContext.getBeanNamesForType(ResourceAssignmentProcessor::class.java)
 121             .filter { it.startsWith(ResourceResolutionConstants.PREFIX_RESOURCE_RESOLUTION_PROCESSOR) }
 122             .map { it.substringAfter(ResourceResolutionConstants.PREFIX_RESOURCE_RESOLUTION_PROCESSOR) }
 123     }
 124
 125     override suspend fun resolveFromDatabase(
 126         bluePrintRuntimeService: BluePrintRuntimeService<*>,
 127         artifactTemplate: String,
 128         resolutionKey: String
 129     ): String {
 130         return templateResolutionDBService.findByResolutionKeyAndBlueprintNameAndBlueprintVersionAndArtifactName(
 131             bluePrintRuntimeService,
 132             artifactTemplate,
 133             resolutionKey
 134         )
 135     }
 136
 137     override suspend fun resolveResolutionKeysFromDatabase(
 138         bluePrintRuntimeService: BluePrintRuntimeService<*>,
 139         artifactTemplate: String
 140     ): List<String> {
 141         return templateResolutionDBService.findResolutionKeysByBlueprintNameAndBlueprintVersionAndArtifactName(
 142             bluePrintRuntimeService,
 143             artifactTemplate
 144         )
 145     }
 146
 147     override suspend fun resolveArtifactNamesAndResolutionKeysFromDatabase(
 148         bluePrintRuntimeService: BluePrintRuntimeService<*>
 149     ): Map<String, List<String>> {
 150         return templateResolutionDBService.findArtifactNamesAndResolutionKeysByBlueprintNameAndBlueprintVersion(
 151             bluePrintRuntimeService
 152         )
 153     }
 154
 155     override suspend fun resolveResources(
 156         bluePrintRuntimeService: BluePrintRuntimeService<*>,
 157         nodeTemplateName: String,
 158         artifactNames: List<String>,
 159         properties: Map<String, Any>,
 160         stepName: String
 161     ): ResourceResolutionResult {
 162
 163         val resourceAssignmentRuntimeService =
 164             ResourceAssignmentUtils.transformToRARuntimeService(bluePrintRuntimeService, artifactNames.toString())
 165
 166         val templateMap: MutableMap<String, String> = hashMapOf()
 167         val assignmentMap: MutableMap<String, JsonNode> = hashMapOf()
 168         artifactNames.forEach { artifactName ->
 169             val (resolvedStringContent, resourceAssignmentList) = resolveResources(
 170                 resourceAssignmentRuntimeService, nodeTemplateName,
 171                 artifactName, properties
 172             )
 173             val resolvedJsonContent = resourceAssignmentList
 174                 .associateBy({ it.name }, { it.property?.value })
 175                 .asJsonNode()
 176
 177             templateMap[artifactName] = resolvedStringContent
 178             assignmentMap[artifactName] = resolvedJsonContent
 179
 180             val failedResolution = resourceAssignmentList.filter { it.status != "success" && it.property?.required == true }.map { it.name }
 181             if (failedResolution.isNotEmpty()) {
 182                 val errorMessages = mutableListOf("Failed to resolve required values: $failedResolution").apply {
 183                     this.addAll(resourceAssignmentRuntimeService.getBluePrintError().allErrors())
 184                 }
 185                 bluePrintRuntimeService.getBluePrintError().addErrors(stepName, errorMessages)
 186             }
 187         }
 188         return ResourceResolutionResult(templateMap, assignmentMap)
 189     }
 190
 191     override suspend fun resolveResources(
 192         bluePrintRuntimeService: BluePrintRuntimeService<*>,
 193         nodeTemplateName: String,
 194         artifactPrefix: String,
 195         properties: Map<String, Any>
 196     ): Pair<String, MutableList<ResourceAssignment>> {
 197
 198         // Template Artifact Definition Name
 199         val artifactTemplate = "$artifactPrefix-template"
 200         // Resource Assignment Artifact Definition Name
 201         val artifactMapping = "$artifactPrefix-mapping"
 202
 203         log.info("Resolving resource with resource assignment artifact($artifactMapping)")
 204
 205         val resourceAssignmentContent =
 206             bluePrintRuntimeService.resolveNodeTemplateArtifact(nodeTemplateName, artifactMapping)
 207
 208         val resourceAssignments: MutableList<ResourceAssignment> =
 209             JacksonUtils.getListFromJson(resourceAssignmentContent, ResourceAssignment::class.java)
 210                 as? MutableList<ResourceAssignment>
 211                 ?: throw BluePrintProcessorException("couldn't get Dictionary Definitions")
 212
 213         if (isToStore(properties)) {
 214             val existingResourceResolution = isNewResolution(bluePrintRuntimeService, properties, artifactPrefix)
 215             if (existingResourceResolution.isNotEmpty()) {
 216                 updateResourceAssignmentWithExisting(
 217                     bluePrintRuntimeService as ResourceAssignmentRuntimeService,
 218                     existingResourceResolution, resourceAssignments
 219                 )
 220             }
 221         }
 222
 223         // Get the Resource Dictionary Name
 224         val resourceDefinitions: MutableMap<String, ResourceDefinition> = ResourceAssignmentUtils
 225             .resourceDefinitions(bluePrintRuntimeService.bluePrintContext().rootPath)
 226
 227         // Resolve resources
 228         resolveResourceAssignments(
 229             bluePrintRuntimeService,
 230             resourceDefinitions,
 231             resourceAssignments,
 232             artifactPrefix,
 233             properties
 234         )
 235
 236         val resolutionSummary = properties.getOrDefault(
 237             ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOLUTION_SUMMARY,
 238             false
 239         ) as Boolean
 240
 241         val resolvedParamJsonContent =
 242             ResourceAssignmentUtils.generateResourceDataForAssignments(resourceAssignments.toList())
 243         val artifactTemplateDefinition =
 244             bluePrintRuntimeService.bluePrintContext().checkNodeTemplateArtifact(nodeTemplateName, artifactTemplate)
 245
 246         val resolvedContent = when {
 247             artifactTemplateDefinition != null -> {
 248                 blueprintTemplateService.generateContent(
 249                     bluePrintRuntimeService, nodeTemplateName,
 250                     artifactTemplate, resolvedParamJsonContent, false,
 251                     mutableMapOf(
 252                         ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_OCCURRENCE to
 253                             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_OCCURRENCE]
 254                                 .asJsonPrimitive()
 255                     )
 256                 )
 257             }
 258             resolutionSummary -> {
 259                 ResourceAssignmentUtils.generateResolutionSummaryData(resourceAssignments, resourceDefinitions)
 260             }
 261             else -> {
 262                 resolvedParamJsonContent
 263             }
 264         }
 265
 266         if (isToStore(properties)) {
 267             templateResolutionDBService.write(properties, resolvedContent, bluePrintRuntimeService, artifactPrefix)
 268             log.info("Template resolution saved into database successfully : ($properties)")
 269         }
 270
 271         return Pair(resolvedContent, resourceAssignments)
 272     }
 273
 274     override suspend fun resolveResourceDefinition(
 275         blueprintRuntimeService: BluePrintRuntimeService<*>,
 276         resourceDefinitions: MutableMap<String, ResourceDefinition>,
 277         resolveDefinition: String,
 278         sources: List<String>
 279     ): MutableMap<String, JsonNode> {
 280
 281         // Populate Dummy Resource Assignments
 282         val resourceAssignments = createResourceAssignments(resourceDefinitions, resolveDefinition, sources)
 283
 284         resolveResourceAssignments(
 285             blueprintRuntimeService, resourceDefinitions, resourceAssignments,
 286             UUID.randomUUID().toString(), hashMapOf()
 287         )
 288
 289         // Get the data from Resource Assignments
 290         return ResourceAssignmentUtils.generateResourceForAssignments(resourceAssignments)
 291     }
 292
 293     /**
 294      * Iterate the Batch, get the Resource Assignment, dictionary Name, Look for the Resource definition for the
 295      * name, then get the type of the Resource Definition, Get the instance for the Resource Type and process the
 296      * request.
 297      */
 298     override suspend fun resolveResourceAssignments(
 299         blueprintRuntimeService: BluePrintRuntimeService<*>,
 300         resourceDefinitions: MutableMap<String, ResourceDefinition>,
 301         resourceAssignments: MutableList<ResourceAssignment>,
 302         artifactPrefix: String,
 303         properties: Map<String, Any>
 304     ) {
 305
 306         val bulkSequenced = BulkResourceSequencingUtils.process(resourceAssignments)
 307
 308         // Check the BlueprintRuntime Service Should be ResourceAssignmentRuntimeService
 309         val resourceAssignmentRuntimeService = if (blueprintRuntimeService !is ResourceAssignmentRuntimeService) {
 310             ResourceAssignmentUtils.transformToRARuntimeService(blueprintRuntimeService, artifactPrefix)
 311         } else {
 312             blueprintRuntimeService
 313         }
 314
 315         exposeOccurrencePropertyInResourceAssignments(resourceAssignmentRuntimeService, properties)
 316
 317         coroutineScope {
 318             bulkSequenced.forEach { batchResourceAssignments ->
 319                 // Execute Non Dependent Assignments in parallel ( ie asynchronously )
 320                 val deferred = batchResourceAssignments
 321                     .filter { it.name != "*" && it.name != "start" }
 322                     .filter { it.status != BluePrintConstants.STATUS_SUCCESS }
 323                     .map { resourceAssignment ->
 324                         async {
 325                             val dictionaryName = resourceAssignment.dictionaryName
 326                             val dictionarySource = resourceAssignment.dictionarySource
 327
 328                             val processorName = processorName(dictionaryName!!, dictionarySource!!, resourceDefinitions)
 329
 330                             val resourceAssignmentProcessor =
 331                                 applicationContext.getBean(processorName) as? ResourceAssignmentProcessor
 332                                     ?: throw BluePrintProcessorException(
 333                                         "failed to get resource processor ($processorName) " +
 334                                             "for resource assignment(${resourceAssignment.name})"
 335                                     )
 336                             try {
 337                                 // Set BluePrint Runtime Service
 338                                 resourceAssignmentProcessor.raRuntimeService = resourceAssignmentRuntimeService
 339                                 // Set Resource Dictionaries
 340                                 resourceAssignmentProcessor.resourceDictionaries = resourceDefinitions
 341
 342                                 resourceAssignmentProcessor.resourceAssignments = resourceAssignments
 343
 344                                 // Invoke Apply Method
 345                                 resourceAssignmentProcessor.applyNB(resourceAssignment)
 346
 347                                 if (isToStore(properties)) {
 348                                     resourceResolutionDBService.write(
 349                                         properties,
 350                                         blueprintRuntimeService,
 351                                         artifactPrefix,
 352                                         resourceAssignment
 353                                     )
 354                                     log.info("Resource resolution saved into database successfully : (${resourceAssignment.name})")
 355                                 }
 356
 357                                 // Set errors from RA
 358                                 blueprintRuntimeService.setBluePrintError(resourceAssignmentRuntimeService.getBluePrintError())
 359                             } catch (e: RuntimeException) {
 360                                 log.error("Fail in processing ${resourceAssignment.name}", e)
 361                                 throw BluePrintProcessorException(e)
 362                             }
 363                         }
 364                     }
 365                 log.debug("Resolving (${deferred.size})resources parallel.")
 366                 deferred.awaitAll()
 367             }
 368         }
 369     }
 370
 371     /**
 372      * If the Source instance is "input", then it is not mandatory to have source Resource Definition, So it can
 373      *  derive the default input processor.
 374      */
 375     private fun processorName(
 376         dictionaryName: String,
 377         dictionarySource: String,
 378         resourceDefinitions: MutableMap<String, ResourceDefinition>
 379     ): String {
 380         val processorName: String = when (dictionarySource) {
 381             "input" -> {
 382                 "${ResourceResolutionConstants.PREFIX_RESOURCE_RESOLUTION_PROCESSOR}source-input"
 383             }
 384             "default" -> {
 385                 "${ResourceResolutionConstants.PREFIX_RESOURCE_RESOLUTION_PROCESSOR}source-default"
 386             }
 387             else -> {
 388                 val resourceDefinition = resourceDefinitions[dictionaryName]
 389                     ?: throw BluePrintProcessorException("couldn't get resource dictionary definition for $dictionaryName")
 390
 391                 val resourceSource = resourceDefinition.sources[dictionarySource]
 392                     ?: throw BluePrintProcessorException("couldn't get resource definition $dictionaryName source($dictionarySource)")
 393
 394                 ResourceResolutionConstants.PREFIX_RESOURCE_RESOLUTION_PROCESSOR.plus(resourceSource.type)
 395             }
 396         }
 397         checkNotEmpty(processorName) {
 398             "couldn't get processor name for resource dictionary definition($dictionaryName) source($dictionarySource)"
 399         }
 400
 401         return processorName
 402     }
 403
 404     // Check whether to store or not the resolution of resource and template
 405     private fun isToStore(properties: Map<String, Any>): Boolean {
 406         return properties.containsKey(ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_STORE_RESULT) &&
 407             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_STORE_RESULT] as Boolean
 408     }
 409
 410     // Check whether resolution already exist in the database for the specified resolution-key or resourceId/resourceType
 411     private suspend fun isNewResolution(
 412         bluePrintRuntimeService: BluePrintRuntimeService<*>,
 413         properties: Map<String, Any>,
 414         artifactPrefix: String
 415     ): List<ResourceResolution> {
 416         val occurrence = properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_OCCURRENCE] as Int
 417         val resolutionKey = properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOLUTION_KEY] as String
 418         val resourceId = properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOURCE_ID] as String
 419         val resourceType = properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOURCE_TYPE] as String
 420
 421         if (resolutionKey.isNotEmpty()) {
 422             val existingResourceAssignments =
 423                 resourceResolutionDBService.findByBlueprintNameAndBlueprintVersionAndArtifactNameAndResolutionKeyAndOccurrence(
 424                     bluePrintRuntimeService,
 425                     resolutionKey,
 426                     occurrence,
 427                     artifactPrefix
 428                 )
 429             if (existingResourceAssignments.isNotEmpty()) {
 430                 log.info(
 431                     "Resolution with resolutionKey=($resolutionKey) already exist - will resolve all resources not already resolved.",
 432                     resolutionKey
 433                 )
 434             }
 435             return existingResourceAssignments
 436         } else if (resourceId.isNotEmpty() && resourceType.isNotEmpty()) {
 437             val existingResourceAssignments =
 438                 resourceResolutionDBService.findByBlueprintNameAndBlueprintVersionAndArtifactNameAndResourceIdAndResourceTypeAndOccurrence(
 439                     bluePrintRuntimeService,
 440                     resourceId,
 441                     resourceType,
 442
 443                     occurrence,
 444                     artifactPrefix
 445                 )
 446             if (existingResourceAssignments.isNotEmpty()) {
 447                 log.info(
 448                     "Resolution with resourceId=($resourceId) and resourceType=($resourceType) already " +
 449                         "exist - will resolve all resources not already resolved."
 450                 )
 451             }
 452             return existingResourceAssignments
 453         }
 454         return emptyList()
 455     }
 456
 457     // Update the resource assignment list with the status of the resource that have already been resolved
 458     private fun updateResourceAssignmentWithExisting(
 459         raRuntimeService: ResourceAssignmentRuntimeService,
 460         resourceResolutionList: List<ResourceResolution>,
 461         resourceAssignmentList: MutableList<ResourceAssignment>
 462     ) {
 463         resourceResolutionList.forEach { resourceResolution ->
 464             if (resourceResolution.status == BluePrintConstants.STATUS_SUCCESS) {
 465                 resourceAssignmentList.forEach {
 466                     if (compareOne(resourceResolution, it)) {
 467                         log.info(
 468                             "Resource ({}) already resolved: value=({})", it.name,
 469                             if (hasLogProtect(it.property)) LOG_REDACTED else resourceResolution.value
 470                         )
 471
 472                         // Make sure to recreate value as per the defined type.
 473                         val value = resourceResolution.value!!.asJsonType(it.property!!.type)
 474                         it.property!!.value = value
 475                         it.status = resourceResolution.status
 476                         ResourceAssignmentUtils.setResourceDataValue(it, raRuntimeService, value)
 477                     }
 478                 }
 479             }
 480         }
 481     }
 482
 483     // Comparision between what we have in the database vs what we have to assign.
 484     private fun compareOne(resourceResolution: ResourceResolution, resourceAssignment: ResourceAssignment): Boolean {
 485         return (
 486             resourceResolution.name == resourceAssignment.name &&
 487                 resourceResolution.dictionaryName == resourceAssignment.dictionaryName &&
 488                 resourceResolution.dictionarySource == resourceAssignment.dictionarySource &&
 489                 resourceResolution.dictionaryVersion == resourceAssignment.version
 490             )
 491     }
 492
 493     private fun exposeOccurrencePropertyInResourceAssignments(
 494         raRuntimeService: ResourceAssignmentRuntimeService,
 495         properties: Map<String, Any>
 496     ) {
 497         raRuntimeService.putResolutionStore(
 498             ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_OCCURRENCE,
 499             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_OCCURRENCE].asJsonPrimitive()
 500         )
 501     }
 502 }