From 37f4fdcc1f78c77dd20e3a66c4e438b5ed09a5f2 Mon Sep 17 00:00:00 2001 From: Steve Siani Date: Tue, 6 Aug 2019 17:59:20 -0400 Subject: [PATCH] Vault Client Rest API implementation for sensitive data Issue-ID: CCSDK-1597 Signed-off-by: Steve Siani Change-Id: I81bf24d0dd0a12b0bb3a038647773900ad80e7fd --- .../starter-dictionary/vault-secrets.json | 36 ++++++++ .../src/main/resources/application-dev.properties | 2 +- .../src/main/resources/application.properties | 2 +- .../resolution/ResourceResolutionConstants.kt | 1 + .../processor/RestResourceResolutionProcessor.kt | 9 +- .../resolution/utils/ResourceAssignmentUtils.kt | 98 +++++++++++++++------- 6 files changed, 115 insertions(+), 33 deletions(-) create mode 100644 components/model-catalog/resource-dictionary/starter-dictionary/vault-secrets.json diff --git a/components/model-catalog/resource-dictionary/starter-dictionary/vault-secrets.json b/components/model-catalog/resource-dictionary/starter-dictionary/vault-secrets.json new file mode 100644 index 000000000..6b4d6a49d --- /dev/null +++ b/components/model-catalog/resource-dictionary/starter-dictionary/vault-secrets.json @@ -0,0 +1,36 @@ +{ + "tags": "vault-secrets", + "name": "vault-secrets", + "property": { + "description": "Vault secret resolution with dynamic endpoint-selector", + "type": "map", + "entry_schema": { + "type": "string" + } + }, + "updated-by": "Steve Siani ", + "sources": { + "input": { + "type": "source-input" + }, + "vault-data": { + "type": "source-rest", + "properties": { + "endpoint-selector": "dynamic-secret-source", + "verb": "GET", + "type": "JSON", + "url-path": "$secret-path", + "path": "", + "input-key-mapping": { + "secret-path": "secret-path" + }, + "output-key-mapping": { + "data": "data" + }, + "key-dependencies": [ + "secret-path" + ] + } + } + } +} \ No newline at end of file diff --git a/ms/blueprintsprocessor/application/src/main/resources/application-dev.properties b/ms/blueprintsprocessor/application/src/main/resources/application-dev.properties index 2fd595102..3ae9414a9 100755 --- a/ms/blueprintsprocessor/application/src/main/resources/application-dev.properties +++ b/ms/blueprintsprocessor/application/src/main/resources/application-dev.properties @@ -26,7 +26,7 @@ server.port=8081 ### START -Controller Blueprints Properties # Load Resource Source Mappings -resourceSourceMappings=processor-db=source-db,input=source-input,default=source-default,sdnc=source-rest,aai-data=source-rest,capability=source-capability +resourceSourceMappings=processor-db=source-db,input=source-input,default=source-default,sdnc=source-rest,aai-data=source-rest,capability=source-capability,vault-data=source-rest # Controller Blueprints Core Configuration blueprintsprocessor.blueprintDeployPath=blueprints/deploy diff --git a/ms/blueprintsprocessor/application/src/main/resources/application.properties b/ms/blueprintsprocessor/application/src/main/resources/application.properties index 37eb87c13..8fe00e709 100755 --- a/ms/blueprintsprocessor/application/src/main/resources/application.properties +++ b/ms/blueprintsprocessor/application/src/main/resources/application.properties @@ -16,7 +16,7 @@ # Web server config ### START -Controller Blueprints Properties # Load Resource Source Mappings -resourceSourceMappings=processor-db=source-db,input=source-input,default=source-default,sdnc=source-rest,aai-data=source-rest,capability=source-capability,rest=source-rest +resourceSourceMappings=processor-db=source-db,input=source-input,default=source-default,sdnc=source-rest,aai-data=source-rest,capability=source-capability,rest=source-rest,vault-data=source-rest # Controller Blueprints Core Configuration blueprintsprocessor.blueprintDeployPath=/opt/app/onap/blueprints/deploy diff --git a/ms/blueprintsprocessor/functions/resource-resolution/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/resource/resolution/ResourceResolutionConstants.kt b/ms/blueprintsprocessor/functions/resource-resolution/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/resource/resolution/ResourceResolutionConstants.kt index 769644288..2a9218df3 100644 --- a/ms/blueprintsprocessor/functions/resource-resolution/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/resource/resolution/ResourceResolutionConstants.kt +++ b/ms/blueprintsprocessor/functions/resource-resolution/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/resource/resolution/ResourceResolutionConstants.kt @@ -29,4 +29,5 @@ object ResourceResolutionConstants { const val RESOURCE_RESOLUTION_INPUT_OCCURRENCE = "occurrence" const val RESOURCE_RESOLUTION_INPUT_RESOURCE_ID = "resource-id" const val RESOURCE_RESOLUTION_INPUT_RESOURCE_TYPE = "resource-type" + val DATA_DICTIONARY_SECRET_SOURCE_TYPES = arrayOf("vault-data") //Add more secret data dictionary source type here } \ No newline at end of file diff --git a/ms/blueprintsprocessor/functions/resource-resolution/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/resource/resolution/processor/RestResourceResolutionProcessor.kt b/ms/blueprintsprocessor/functions/resource-resolution/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/resource/resolution/processor/RestResourceResolutionProcessor.kt index da156096e..6515b11cd 100644 --- a/ms/blueprintsprocessor/functions/resource-resolution/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/resource/resolution/processor/RestResourceResolutionProcessor.kt +++ b/ms/blueprintsprocessor/functions/resource-resolution/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/resource/resolution/processor/RestResourceResolutionProcessor.kt @@ -19,6 +19,7 @@ package org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.pro import com.fasterxml.jackson.databind.node.MissingNode import com.fasterxml.jackson.databind.node.NullNode +import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.ResourceResolutionConstants.DATA_DICTIONARY_SECRET_SOURCE_TYPES import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.ResourceResolutionConstants.PREFIX_RESOURCE_RESOLUTION_PROCESSOR import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.RestResourceSource import org.onap.ccsdk.cds.blueprintsprocessor.functions.resource.resolution.utils.ResourceAssignmentUtils @@ -135,8 +136,12 @@ open class RestResourceResolutionProcessor(private val blueprintRestLibPropertyS val responseNode = checkNotNull(JacksonUtils.jsonNode(restResponse).at(path)) { "Failed to find path ($path) in response ($restResponse)" } - logger.info("populating value for output mapping ($outputKeyMapping), from json ($responseNode)") - + if (resourceAssignment.dictionarySource in DATA_DICTIONARY_SECRET_SOURCE_TYPES) { + logger.info("populating value for output mapping ($outputKeyMapping), from json (*************)") + } + else { + logger.info("populating value for output mapping ($outputKeyMapping), from json ($responseNode)") + } val parsedResponseNode = ResourceAssignmentUtils.parseResponseNode(responseNode, resourceAssignment, raRuntimeService, outputKeyMapping) diff --git a/ms/blueprintsprocessor/functions/resource-resolution/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/resource/resolution/utils/ResourceAssignmentUtils.kt b/ms/blueprintsprocessor/functions/resource-resolution/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/resource/resolution/utils/ResourceAssignmentUtils.kt index 117df1e5b..819246b99 100644 --- a/ms/blueprintsprocessor/functions/resource-resolution/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/resource/resolution/utils/ResourceAssignmentUtils.kt +++ b/ms/blueprintsprocessor/functions/resource-resolution/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/resource/resolution/utils/ResourceAssignmentUtils.kt @@ -71,9 +71,15 @@ class ResourceAssignmentUtils { try { if (resourceProp.type.isNotEmpty()) { - logger.info("Setting Resource Value ($value) for Resource Name " + - "(${resourceAssignment.name}), definition(${resourceAssignment.dictionaryName}) " + - "of type (${resourceProp.type})") + if (resourceAssignment.dictionarySource in ResourceResolutionConstants.DATA_DICTIONARY_SECRET_SOURCE_TYPES) { + logger.info("Setting Resource Value (*********) for Resource Name " + + "(${resourceAssignment.name}) of type (${resourceProp.type})") + } + else { + logger.info("Setting Resource Value ($value) for Resource Name " + + "(${resourceAssignment.name}), definition(${resourceAssignment.dictionaryName}) " + + "of type (${resourceProp.type})") + } setResourceValue(resourceAssignment, raRuntimeService, value) resourceAssignment.updatedDate = Date() resourceAssignment.updatedBy = BluePrintConstants.USER_SYSTEM @@ -122,17 +128,29 @@ class ResourceAssignmentUtils { val mapper = ObjectMapper() val root: ObjectNode = mapper.createObjectNode() + var containsSecret = false assignments.forEach { if (isNotEmpty(it.name) && it.property != null) { val rName = it.name val type = nullToEmpty(it.property?.type).toLowerCase() val value = useDefaultValueIfNull(it, rName) - logger.info("Generating Resource name ($rName), type ($type), value ($value)") + if (it.dictionarySource in ResourceResolutionConstants.DATA_DICTIONARY_SECRET_SOURCE_TYPES) { + logger.info("Generating Resource name ($rName), type ($type), value (************)") + containsSecret = true + } + else { + logger.info("Generating Resource name ($rName), type ($type), value ($value)") + } root.set(rName, value) } } result = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(root) - logger.info("Generated Resource Param Data ($result)") + if (containsSecret) { + logger.info("Generated Resource Param Data (***********)") + } + else{ + logger.info("Generated Resource Param Data ($result)") + } } catch (e: Exception) { throw BluePrintProcessorException("Resource Assignment is failed with $e.message", e) } @@ -148,7 +166,12 @@ class ResourceAssignmentUtils { val rName = it.name val type = nullToEmpty(it.property?.type).toLowerCase() val value = useDefaultValueIfNull(it, rName) - logger.trace("Generating Resource name ($rName), type ($type), value ($value)") + if (it.dictionarySource in ResourceResolutionConstants.DATA_DICTIONARY_SECRET_SOURCE_TYPES) { + logger.trace("Generating Resource name ($rName), type ($type), value (************)") + } + else { + logger.trace("Generating Resource name ($rName), type ($type), value ($value)") + } data[rName] = value } } @@ -200,9 +223,17 @@ class ResourceAssignmentUtils { throw BluePrintProcessorException("Couldn't get data dictionary type for dictionary name (${resourceAssignment.name})") } val type = resourceAssignment.property!!.type + + if (resourceAssignment.dictionarySource in ResourceResolutionConstants.DATA_DICTIONARY_SECRET_SOURCE_TYPES) { + logger.info("For template key (${resourceAssignment.name}) setting value as (***************)") + } + else { + logger.info("For template key (${resourceAssignment.name}) setting value as ($responseNode)") + } + return when (type) { in BluePrintTypes.validPrimitiveTypes() -> { - parseResponseNodeForPrimitiveTypes(responseNode, resourceAssignment, outputKeyMapping) + parseResponseNodeForPrimitiveTypes(responseNode, outputKeyMapping) } in BluePrintTypes.validCollectionTypes() -> { // Array Types @@ -219,11 +250,8 @@ class ResourceAssignmentUtils { } } - private fun parseResponseNodeForPrimitiveTypes(responseNode: JsonNode, resourceAssignment: ResourceAssignment, + private fun parseResponseNodeForPrimitiveTypes(responseNode: JsonNode, outputKeyMapping: MutableMap): JsonNode { - val dName = resourceAssignment.dictionaryName - logger.info("For template key (${resourceAssignment.name}) setting value as ($responseNode)") - var result: JsonNode? = responseNode if (responseNode.isComplexType()) { val key = outputKeyMapping.keys.firstOrNull() @@ -264,6 +292,7 @@ class ResourceAssignmentUtils { raRuntimeService: ResourceAssignmentRuntimeService, outputKeyMapping: MutableMap): JsonNode { val dName = resourceAssignment.dictionaryName + val dSource = resourceAssignment.dictionarySource if ((resourceAssignment.property?.entrySchema?.type).isNullOrEmpty()) { throw BluePrintProcessorException("Couldn't get data type for dictionary type " + "(${resourceAssignment.property!!.type}) and dictionary name ($dName)") @@ -278,13 +307,14 @@ class ResourceAssignmentUtils { val responseArrayNode = responseNode.toList() for (responseSingleJsonNode in responseArrayNode) { val arrayChildNode = parseArrayNodeElementWithOutputKeyMapping(raRuntimeService, responseSingleJsonNode, - outputKeyMapping, entrySchemaType) + outputKeyMapping, entrySchemaType, dSource!!) arrayNode.add(arrayChildNode) } } is ObjectNode -> { val responseArrayNode = responseNode.rootFieldsToMap() - val arrayNodeResult = parseObjectNodeWithOutputKeyMapping(responseArrayNode, outputKeyMapping, entrySchemaType) + val arrayNodeResult = parseObjectNodeWithOutputKeyMapping(responseArrayNode, + outputKeyMapping, entrySchemaType, dSource!!) arrayNode.addAll(arrayNodeResult) } else -> { @@ -312,9 +342,6 @@ class ResourceAssignmentUtils { } } } - - logger.info("For template key (${resourceAssignment.name}) setting value as ($arrayNode)") - return arrayNode } @@ -322,12 +349,13 @@ class ResourceAssignmentUtils { raRuntimeService: ResourceAssignmentRuntimeService, outputKeyMapping: MutableMap): JsonNode { val entrySchemaType = resourceAssignment.property!!.type + val dictionarySource = resourceAssignment.dictionarySource!! val dictionaryName = resourceAssignment.dictionaryName!! var result: ObjectNode if (checkOutputKeyMappingInDataTypeProperties(entrySchemaType, outputKeyMapping, raRuntimeService)) { - result = parseArrayNodeElementWithOutputKeyMapping(raRuntimeService, responseNode, outputKeyMapping, entrySchemaType) + result = parseArrayNodeElementWithOutputKeyMapping(raRuntimeService, responseNode, outputKeyMapping, entrySchemaType, dictionarySource!!) } else { val childNode = JacksonUtils.objectMapper.createObjectNode() @@ -340,6 +368,8 @@ class ResourceAssignmentUtils { NullNode.getInstance() } + logKeyValueResolvedResource(it.key, responseKeyValue, entrySchemaType, dictionarySource) + JacksonUtils.populateJsonNodeValues(it.value, responseKeyValue, entrySchemaType, childNode) } @@ -354,7 +384,8 @@ class ResourceAssignmentUtils { private fun parseArrayNodeElementWithOutputKeyMapping(raRuntimeService: ResourceAssignmentRuntimeService, responseSingleJsonNode: JsonNode, outputKeyMapping: - MutableMap, entrySchemaType: String): ObjectNode { + MutableMap, entrySchemaType: String, + dictionarySource: String): ObjectNode { val arrayChildNode = JacksonUtils.objectMapper.createObjectNode() outputKeyMapping.map { @@ -367,31 +398,29 @@ class ResourceAssignmentUtils { val propertyTypeForDataType = ResourceAssignmentUtils .getPropertyType(raRuntimeService, entrySchemaType, it.key) - logger.info("For List Type Resource: key (${it.key}), value ($responseKeyValue), " + - "type ({$propertyTypeForDataType})") + logKeyValueResolvedResource(it.key, responseKeyValue, propertyTypeForDataType, dictionarySource) JacksonUtils.populateJsonNodeValues(it.value, responseKeyValue, propertyTypeForDataType, arrayChildNode) } - return arrayChildNode } private fun parseObjectNodeWithOutputKeyMapping(responseArrayNode: MutableMap, outputKeyMapping: MutableMap, - entrySchemaType: String): ArrayNode { + entrySchemaType: String, + dictionarySource: String): ArrayNode { val arrayNode = JacksonUtils.objectMapper.createArrayNode() outputKeyMapping.map { val objectNode = JacksonUtils.objectMapper.createObjectNode() val responseSingleJsonNode = responseArrayNode.filterKeys { key -> key == it.key }.entries.firstOrNull() - if (responseSingleJsonNode == null) { - JacksonUtils.populateJsonNodeValues(it.value, NullNode.getInstance(), entrySchemaType, objectNode) - } - else - { - JacksonUtils.populateJsonNodeValues(it.value, responseSingleJsonNode.value, entrySchemaType, objectNode) - } + val responseNode = responseSingleJsonNode?.value ?: NullNode.getInstance() + + logKeyValueResolvedResource(it.key, responseNode, entrySchemaType, dictionarySource) + + JacksonUtils.populateJsonNodeValues(it.value, responseNode, entrySchemaType, objectNode) + arrayNode.add(objectNode) } @@ -404,5 +433,16 @@ class ResourceAssignmentUtils { val result = outputKeyMapping.filterKeys { !dataTypeProps!!.containsKey(it) }.keys.firstOrNull() return result == null } + + private fun logKeyValueResolvedResource(key: String, value: JsonNode, type: String, dictionarySource: String) { + if (dictionarySource in ResourceResolutionConstants.DATA_DICTIONARY_SECRET_SOURCE_TYPES) { + logger.info("For List Type Resource: key ($key), value (****************), " + + "type ({$type})") + } + else { + logger.info("For List Type Resource: key ($key), value ($value), " + + "type ({$type})") + } + } } } \ No newline at end of file -- 2.16.6