Enable force-resolution
[ccsdk/cds.git] / ms / blueprintsprocessor / functions / resource-resolution / src / main / kotlin / org / onap / ccsdk / cds / blueprintsprocessor / functions / resource / resolution / ResourceResolutionService.kt
1 /*
2  *  Copyright © 2017-2018 AT&T Intellectual Property.
3  *  Modifications Copyright © 2018-2019 IBM, Bell Canada
4  *
5  *  Licensed under the Apache License, Version 2.0 (the "License");
6  *  you may not use this file except in compliance with the License.
7  *  You may obtain a copy of the License at
8  *
9  *      http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *  Unless required by applicable law or agreed to in writing, software
12  *  distributed under the License is distributed on an "AS IS" BASIS,
13  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *  See the License for the specific language governing permissions and
15  *  limitations under the License.
16  */
17
18 package org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution
19
20 import com.fasterxml.jackson.databind.JsonNode
21 import kotlinx.coroutines.async
22 import kotlinx.coroutines.awaitAll
23 import kotlinx.coroutines.coroutineScope
24 import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.db.ResourceResolution
25 import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.db.ResourceResolutionDBService
26 import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.db.TemplateResolutionService
27 import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.processor.ResourceAssignmentProcessor
28 import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.utils.ResourceAssignmentUtils
29 import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.utils.ResourceDefinitionUtils.createResourceAssignments
30 import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintConstants
31 import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintProcessorException
32 import org.onap.ccsdk.cds.controllerblueprints.core.asJsonNode
33 import org.onap.ccsdk.cds.controllerblueprints.core.asJsonPrimitive
34 import org.onap.ccsdk.cds.controllerblueprints.core.asJsonType
35 import org.onap.ccsdk.cds.controllerblueprints.core.checkNotEmpty
36 import org.onap.ccsdk.cds.controllerblueprints.core.common.ApplicationConstants.LOG_REDACTED
37 import org.onap.ccsdk.cds.controllerblueprints.core.service.BluePrintRuntimeService
38 import org.onap.ccsdk.cds.controllerblueprints.core.service.BluePrintTemplateService
39 import org.onap.ccsdk.cds.controllerblueprints.core.utils.JacksonUtils
40 import org.onap.ccsdk.cds.controllerblueprints.core.utils.PropertyDefinitionUtils.Companion.hasLogProtect
41 import org.onap.ccsdk.cds.controllerblueprints.resource.dict.ResourceAssignment
42 import org.onap.ccsdk.cds.controllerblueprints.resource.dict.ResourceDefinition
43 import org.onap.ccsdk.cds.controllerblueprints.resource.dict.utils.BulkResourceSequencingUtils
44 import org.slf4j.LoggerFactory
45 import org.springframework.context.ApplicationContext
46 import org.springframework.stereotype.Service
47 import java.util.UUID
48
49 data class ResourceResolutionResult(
50     val templateMap: MutableMap<String, String>,
51     val assignmentMap: MutableMap<String, JsonNode>
52 )
53
54 interface ResourceResolutionService {
55
56     fun registeredResourceSources(): List<String>
57
58     suspend fun resolveFromDatabase(
59         bluePrintRuntimeService: BluePrintRuntimeService<*>,
60         artifactTemplate: String,
61         resolutionKey: String
62     ): String
63
64     suspend fun resolveResolutionKeysFromDatabase(
65         bluePrintRuntimeService: BluePrintRuntimeService<*>,
66         artifactTemplate: String
67     ): List<String>
68
69     suspend fun resolveArtifactNamesAndResolutionKeysFromDatabase(
70         bluePrintRuntimeService: BluePrintRuntimeService<*>
71     ): Map<String, List<String>>
72
73     suspend fun resolveResources(
74         bluePrintRuntimeService: BluePrintRuntimeService<*>,
75         nodeTemplateName: String,
76         artifactNames: List<String>,
77         properties: Map<String, Any>,
78         stepName: String
79     ): ResourceResolutionResult
80
81     suspend fun resolveResources(
82         bluePrintRuntimeService: BluePrintRuntimeService<*>,
83         nodeTemplateName: String,
84         artifactPrefix: String,
85         properties: Map<String, Any>
86     ): Pair<String, MutableList<ResourceAssignment>>
87
88     /** Resolve resources for all the sources defined in a particular resource Definition[resolveDefinition]
89      * with other [resourceDefinitions] dependencies for the sources [sources]
90      * Used to get the same resource values from multiple sources. **/
91     suspend fun resolveResourceDefinition(
92         blueprintRuntimeService: BluePrintRuntimeService<*>,
93         resourceDefinitions: MutableMap<String, ResourceDefinition>,
94         resolveDefinition: String,
95         sources: List<String>
96     ):
97         MutableMap<String, JsonNode>
98
99     suspend fun resolveResourceAssignments(
100         blueprintRuntimeService: BluePrintRuntimeService<*>,
101         resourceDefinitions: MutableMap<String, ResourceDefinition>,
102         resourceAssignments: MutableList<ResourceAssignment>,
103         artifactPrefix: String,
104         properties: Map<String, Any>
105     )
106 }
107
108 @Service(ResourceResolutionConstants.SERVICE_RESOURCE_RESOLUTION)
109 open class ResourceResolutionServiceImpl(
110     private var applicationContext: ApplicationContext,
111     private var templateResolutionDBService: TemplateResolutionService,
112     private var blueprintTemplateService: BluePrintTemplateService,
113     private var resourceResolutionDBService: ResourceResolutionDBService
114 ) :
115     ResourceResolutionService {
116
117     private val log = LoggerFactory.getLogger(ResourceResolutionService::class.java)
118
119     override fun registeredResourceSources(): List<String> {
120         return applicationContext.getBeanNamesForType(ResourceAssignmentProcessor::class.java)
121             .filter { it.startsWith(ResourceResolutionConstants.PREFIX_RESOURCE_RESOLUTION_PROCESSOR) }
122             .map { it.substringAfter(ResourceResolutionConstants.PREFIX_RESOURCE_RESOLUTION_PROCESSOR) }
123     }
124
125     override suspend fun resolveFromDatabase(
126         bluePrintRuntimeService: BluePrintRuntimeService<*>,
127         artifactTemplate: String,
128         resolutionKey: String
129     ): String {
130         return templateResolutionDBService.findByResolutionKeyAndBlueprintNameAndBlueprintVersionAndArtifactName(
131             bluePrintRuntimeService,
132             artifactTemplate,
133             resolutionKey
134         )
135     }
136
137     override suspend fun resolveResolutionKeysFromDatabase(
138         bluePrintRuntimeService: BluePrintRuntimeService<*>,
139         artifactTemplate: String
140     ): List<String> {
141         return templateResolutionDBService.findResolutionKeysByBlueprintNameAndBlueprintVersionAndArtifactName(
142             bluePrintRuntimeService,
143             artifactTemplate
144         )
145     }
146
147     override suspend fun resolveArtifactNamesAndResolutionKeysFromDatabase(
148         bluePrintRuntimeService: BluePrintRuntimeService<*>
149     ): Map<String, List<String>> {
150         return templateResolutionDBService.findArtifactNamesAndResolutionKeysByBlueprintNameAndBlueprintVersion(
151             bluePrintRuntimeService
152         )
153     }
154
155     override suspend fun resolveResources(
156         bluePrintRuntimeService: BluePrintRuntimeService<*>,
157         nodeTemplateName: String,
158         artifactNames: List<String>,
159         properties: Map<String, Any>,
160         stepName: String
161     ): ResourceResolutionResult {
162
163         val resourceAssignmentRuntimeService =
164             ResourceAssignmentUtils.transformToRARuntimeService(bluePrintRuntimeService, artifactNames.toString())
165
166         val templateMap: MutableMap<String, String> = hashMapOf()
167         val assignmentMap: MutableMap<String, JsonNode> = hashMapOf()
168         artifactNames.forEach { artifactName ->
169             val (resolvedStringContent, resourceAssignmentList) = resolveResources(
170                 resourceAssignmentRuntimeService, nodeTemplateName,
171                 artifactName, properties
172             )
173             val resolvedJsonContent = resourceAssignmentList
174                 .associateBy({ it.name }, { it.property?.value })
175                 .asJsonNode()
176
177             templateMap[artifactName] = resolvedStringContent
178             assignmentMap[artifactName] = resolvedJsonContent
179
180             val failedResolution = resourceAssignmentList.filter { it.status != "success" && it.property?.required == true }.map { it.name }
181             if (failedResolution.isNotEmpty()) {
182                 val errorMessages = mutableListOf("Failed to resolve required values: $failedResolution").apply {
183                     this.addAll(resourceAssignmentRuntimeService.getBluePrintError().allErrors())
184                 }
185                 bluePrintRuntimeService.getBluePrintError().addErrors(stepName, errorMessages)
186             }
187         }
188         return ResourceResolutionResult(templateMap, assignmentMap)
189     }
190
191     override suspend fun resolveResources(
192         bluePrintRuntimeService: BluePrintRuntimeService<*>,
193         nodeTemplateName: String,
194         artifactPrefix: String,
195         properties: Map<String, Any>
196     ): Pair<String, MutableList<ResourceAssignment>> {
197
198         // Template Artifact Definition Name
199         val artifactTemplate = "$artifactPrefix-template"
200         // Resource Assignment Artifact Definition Name
201         val artifactMapping = "$artifactPrefix-mapping"
202         val forceResolution = isForceResolution(properties)
203
204         log.info("Resolving resource with resource assignment artifact($artifactMapping)")
205
206         val resourceAssignmentContent =
207             bluePrintRuntimeService.resolveNodeTemplateArtifact(nodeTemplateName, artifactMapping)
208
209         val resourceAssignments: MutableList<ResourceAssignment> =
210             JacksonUtils.getListFromJson(resourceAssignmentContent, ResourceAssignment::class.java)
211                 as? MutableList<ResourceAssignment>
212                 ?: throw BluePrintProcessorException("couldn't get Dictionary Definitions")
213
214         if (isToStore(properties)) {
215             val existingResourceResolution = isNewResolution(bluePrintRuntimeService, properties, artifactPrefix)
216             if (existingResourceResolution.isNotEmpty()) {
217                 if (forceResolution) {
218                     resourceResolutionDBService.deleteResourceResolutionList(existingResourceResolution)
219                     log.info("Force resolution is enabled - will resolve all resources.")
220                 } else {
221                     updateResourceAssignmentWithExisting(
222                         bluePrintRuntimeService as ResourceAssignmentRuntimeService,
223                         existingResourceResolution, resourceAssignments
224                     )
225                     log.info("Force resolution is disabled - will resolve all resources not already resolved.")
226                 }
227             }
228         }
229
230         // Get the Resource Dictionary Name
231         val resourceDefinitions: MutableMap<String, ResourceDefinition> = ResourceAssignmentUtils
232             .resourceDefinitions(bluePrintRuntimeService.bluePrintContext().rootPath)
233
234         // Resolve resources
235         resolveResourceAssignments(
236             bluePrintRuntimeService,
237             resourceDefinitions,
238             resourceAssignments,
239             artifactPrefix,
240             properties
241         )
242
243         val resolutionSummary = properties.getOrDefault(
244             ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOLUTION_SUMMARY,
245             false
246         ) as Boolean
247
248         val resolvedParamJsonContent =
249             ResourceAssignmentUtils.generateResourceDataForAssignments(resourceAssignments.toList())
250         val artifactTemplateDefinition =
251             bluePrintRuntimeService.bluePrintContext().checkNodeTemplateArtifact(nodeTemplateName, artifactTemplate)
252
253         val resolvedContent = when {
254             artifactTemplateDefinition != null -> {
255                 blueprintTemplateService.generateContent(
256                     bluePrintRuntimeService, nodeTemplateName,
257                     artifactTemplate, resolvedParamJsonContent, false,
258                     mutableMapOf(
259                         ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_OCCURRENCE to
260                             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_OCCURRENCE]
261                                 .asJsonPrimitive()
262                     )
263                 )
264             }
265             resolutionSummary -> {
266                 ResourceAssignmentUtils.generateResolutionSummaryData(resourceAssignments, resourceDefinitions)
267             }
268             else -> {
269                 resolvedParamJsonContent
270             }
271         }
272
273         if (isToStore(properties)) {
274             templateResolutionDBService.write(properties, resolvedContent, bluePrintRuntimeService, artifactPrefix)
275             log.info("Template resolution saved into database successfully : ($properties)")
276         }
277
278         return Pair(resolvedContent, resourceAssignments)
279     }
280
281     override suspend fun resolveResourceDefinition(
282         blueprintRuntimeService: BluePrintRuntimeService<*>,
283         resourceDefinitions: MutableMap<String, ResourceDefinition>,
284         resolveDefinition: String,
285         sources: List<String>
286     ): MutableMap<String, JsonNode> {
287
288         // Populate Dummy Resource Assignments
289         val resourceAssignments = createResourceAssignments(resourceDefinitions, resolveDefinition, sources)
290
291         resolveResourceAssignments(
292             blueprintRuntimeService, resourceDefinitions, resourceAssignments,
293             UUID.randomUUID().toString(), hashMapOf()
294         )
295
296         // Get the data from Resource Assignments
297         return ResourceAssignmentUtils.generateResourceForAssignments(resourceAssignments)
298     }
299
300     /**
301      * Iterate the Batch, get the Resource Assignment, dictionary Name, Look for the Resource definition for the
302      * name, then get the type of the Resource Definition, Get the instance for the Resource Type and process the
303      * request.
304      */
305     override suspend fun resolveResourceAssignments(
306         blueprintRuntimeService: BluePrintRuntimeService<*>,
307         resourceDefinitions: MutableMap<String, ResourceDefinition>,
308         resourceAssignments: MutableList<ResourceAssignment>,
309         artifactPrefix: String,
310         properties: Map<String, Any>
311     ) {
312
313         val bulkSequenced = BulkResourceSequencingUtils.process(resourceAssignments)
314
315         // Check the BlueprintRuntime Service Should be ResourceAssignmentRuntimeService
316         val resourceAssignmentRuntimeService = if (blueprintRuntimeService !is ResourceAssignmentRuntimeService) {
317             ResourceAssignmentUtils.transformToRARuntimeService(blueprintRuntimeService, artifactPrefix)
318         } else {
319             blueprintRuntimeService
320         }
321
322         exposeOccurrencePropertyInResourceAssignments(resourceAssignmentRuntimeService, properties)
323
324         coroutineScope {
325             bulkSequenced.forEach { batchResourceAssignments ->
326                 // Execute Non Dependent Assignments in parallel ( ie asynchronously )
327                 val deferred = batchResourceAssignments
328                     .filter { it.name != "*" && it.name != "start" }
329                     .filter { it.status != BluePrintConstants.STATUS_SUCCESS }
330                     .map { resourceAssignment ->
331                         async {
332                             val dictionaryName = resourceAssignment.dictionaryName
333                             val dictionarySource = resourceAssignment.dictionarySource
334
335                             val processorName = processorName(dictionaryName!!, dictionarySource!!, resourceDefinitions)
336
337                             val resourceAssignmentProcessor =
338                                 applicationContext.getBean(processorName) as? ResourceAssignmentProcessor
339                                     ?: throw BluePrintProcessorException(
340                                         "failed to get resource processor ($processorName) " +
341                                             "for resource assignment(${resourceAssignment.name})"
342                                     )
343                             try {
344                                 // Set BluePrint Runtime Service
345                                 resourceAssignmentProcessor.raRuntimeService = resourceAssignmentRuntimeService
346                                 // Set Resource Dictionaries
347                                 resourceAssignmentProcessor.resourceDictionaries = resourceDefinitions
348
349                                 resourceAssignmentProcessor.resourceAssignments = resourceAssignments
350
351                                 // Invoke Apply Method
352                                 resourceAssignmentProcessor.applyNB(resourceAssignment)
353
354                                 if (isToStore(properties)) {
355                                     resourceResolutionDBService.write(
356                                         properties,
357                                         blueprintRuntimeService,
358                                         artifactPrefix,
359                                         resourceAssignment
360                                     )
361                                     log.info("Resource resolution saved into database successfully : (${resourceAssignment.name})")
362                                 }
363
364                                 // Set errors from RA
365                                 blueprintRuntimeService.setBluePrintError(resourceAssignmentRuntimeService.getBluePrintError())
366                             } catch (e: RuntimeException) {
367                                 log.error("Fail in processing ${resourceAssignment.name}", e)
368                                 throw BluePrintProcessorException(e)
369                             }
370                         }
371                     }
372                 log.debug("Resolving (${deferred.size})resources parallel.")
373                 deferred.awaitAll()
374             }
375         }
376     }
377
378     /**
379      * If the Source instance is "input", then it is not mandatory to have source Resource Definition, So it can
380      *  derive the default input processor.
381      */
382     private fun processorName(
383         dictionaryName: String,
384         dictionarySource: String,
385         resourceDefinitions: MutableMap<String, ResourceDefinition>
386     ): String {
387         val processorName: String = when (dictionarySource) {
388             "input" -> {
389                 "${ResourceResolutionConstants.PREFIX_RESOURCE_RESOLUTION_PROCESSOR}source-input"
390             }
391             "default" -> {
392                 "${ResourceResolutionConstants.PREFIX_RESOURCE_RESOLUTION_PROCESSOR}source-default"
393             }
394             else -> {
395                 val resourceDefinition = resourceDefinitions[dictionaryName]
396                     ?: throw BluePrintProcessorException("couldn't get resource dictionary definition for $dictionaryName")
397
398                 val resourceSource = resourceDefinition.sources[dictionarySource]
399                     ?: throw BluePrintProcessorException("couldn't get resource definition $dictionaryName source($dictionarySource)")
400
401                 ResourceResolutionConstants.PREFIX_RESOURCE_RESOLUTION_PROCESSOR.plus(resourceSource.type)
402             }
403         }
404         checkNotEmpty(processorName) {
405             "couldn't get processor name for resource dictionary definition($dictionaryName) source($dictionarySource)"
406         }
407
408         return processorName
409     }
410
411     // Check whether to store or not the resolution of resource and template
412     private fun isToStore(properties: Map<String, Any>): Boolean {
413         return properties.containsKey(ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_STORE_RESULT) &&
414             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_STORE_RESULT] as Boolean
415     }
416
417     private fun isForceResolution(properties: Map<String, Any>): Boolean =
418         properties.containsKey(ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_FORCE_RESOLUTION) &&
419             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_FORCE_RESOLUTION] as Boolean
420
421     // Check whether resolution already exist in the database for the specified resolution-key or resourceId/resourceType
422     private suspend fun isNewResolution(
423         bluePrintRuntimeService: BluePrintRuntimeService<*>,
424         properties: Map<String, Any>,
425         artifactPrefix: String
426     ): List<ResourceResolution> {
427         val occurrence = properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_OCCURRENCE] as Int
428         val resolutionKey = properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOLUTION_KEY] as String
429         val resourceId = properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOURCE_ID] as String
430         val resourceType = properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_RESOURCE_TYPE] as String
431
432         if (resolutionKey.isNotEmpty()) {
433             val existingResourceAssignments =
434                 resourceResolutionDBService.findByBlueprintNameAndBlueprintVersionAndArtifactNameAndResolutionKeyAndOccurrence(
435                     bluePrintRuntimeService,
436                     resolutionKey,
437                     occurrence,
438                     artifactPrefix
439                 )
440             if (existingResourceAssignments.isNotEmpty()) {
441                 log.info(
442                     "Resolution with resolutionKey=($resolutionKey) already exist",
443                     resolutionKey
444                 )
445             }
446             return existingResourceAssignments
447         } else if (resourceId.isNotEmpty() && resourceType.isNotEmpty()) {
448             val existingResourceAssignments =
449                 resourceResolutionDBService.findByBlueprintNameAndBlueprintVersionAndArtifactNameAndResourceIdAndResourceTypeAndOccurrence(
450                     bluePrintRuntimeService,
451                     resourceId,
452                     resourceType,
453
454                     occurrence,
455                     artifactPrefix
456                 )
457             if (existingResourceAssignments.isNotEmpty()) {
458                 log.info(
459                     "Resolution with resourceId=($resourceId) and resourceType=($resourceType) already exist"
460                 )
461             }
462             return existingResourceAssignments
463         }
464         return emptyList()
465     }
466
467     // Update the resource assignment list with the status of the resource that have already been resolved
468     private fun updateResourceAssignmentWithExisting(
469         raRuntimeService: ResourceAssignmentRuntimeService,
470         resourceResolutionList: List<ResourceResolution>,
471         resourceAssignmentList: MutableList<ResourceAssignment>
472     ) {
473         resourceResolutionList.forEach { resourceResolution ->
474             if (resourceResolution.status == BluePrintConstants.STATUS_SUCCESS) {
475                 resourceAssignmentList.forEach {
476                     if (compareOne(resourceResolution, it)) {
477                         log.info(
478                             "Resource ({}) already resolved: value=({})", it.name,
479                             if (hasLogProtect(it.property)) LOG_REDACTED else resourceResolution.value
480                         )
481
482                         // Make sure to recreate value as per the defined type.
483                         val value = resourceResolution.value!!.asJsonType(it.property!!.type)
484                         it.property!!.value = value
485                         it.status = resourceResolution.status
486                         ResourceAssignmentUtils.setResourceDataValue(it, raRuntimeService, value)
487                     }
488                 }
489             }
490         }
491     }
492
493     // Comparision between what we have in the database vs what we have to assign.
494     private fun compareOne(resourceResolution: ResourceResolution, resourceAssignment: ResourceAssignment): Boolean {
495         return (
496             resourceResolution.name == resourceAssignment.name &&
497                 resourceResolution.dictionaryName == resourceAssignment.dictionaryName &&
498                 resourceResolution.dictionarySource == resourceAssignment.dictionarySource &&
499                 resourceResolution.dictionaryVersion == resourceAssignment.version
500             )
501     }
502
503     private fun exposeOccurrencePropertyInResourceAssignments(
504         raRuntimeService: ResourceAssignmentRuntimeService,
505         properties: Map<String, Any>
506     ) {
507         raRuntimeService.putResolutionStore(
508             ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_OCCURRENCE,
509             properties[ResourceResolutionConstants.RESOURCE_RESOLUTION_INPUT_OCCURRENCE].asJsonPrimitive()
510         )
511     }
512 }