Integration Test - Component Executor

Change-Id: Icbe745b30c7bbb973bc9bb29bf547340b32f1b0f
Issue-ID: CCSDK-1164
Signed-off-by: Alexis de Talhouët <adetalhouet89@gmail.com>

diff --git a/components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/node_types.json b/components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/node_types.json
index 84d5b5d..d7002cd 100644 (file)
--- a/components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/node_types.json
+++ b/components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/node_types.json
@@ -3,6 +3,12 @@
     "component-remote-python-executor": {
       "description": "This is Remote Python Execution Component.",
       "version": "1.0.0",
+      "attributes": {
+        "execution-logs": {
+          "required": true,
+          "type": "string"
+        }
+      },
       "capabilities": {
         "component-node": {
           "type": "tosca.capabilities.Node"
@@ -17,25 +23,13 @@
                   "description": "Remote Container or Server selector name.",
                   "required": false,
                   "type": "string",
-                  "default": "default"
+                  "default": "remote-python"
                 },
                 "dynamic-properties": {
                   "description": "Dynamic Json Content or DSL Json reference.",
                   "required": false,
                   "type": "json"
                 }
-              },
-              "outputs": {
-                "response-data": {
-                  "description": "Execution Response Data in JSON format.",
-                  "required": false,
-                  "type": "json"
-                },
-                "status": {
-                  "description": "Status of the Component Execution ( success or failure )",
-                  "required": true,
-                  "type": "string"
-                }
               }
             }
           }

diff --git a/components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/remote_scripts.json b/components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/remote_scripts.json
index b978506..7c27901 100644 (file)
--- a/components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/remote_scripts.json
+++ b/components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/remote_scripts.json
@@ -39,7 +39,18 @@
             ]
           }
         },
-        "inputs": {}
+        "inputs": {},
+        "outputs": {
+          "logs": {
+            "type": "json",
+            "value": {
+              "get_attribute": [
+                "execute-remote-python",
+                "execution-logs"
+              ]
+            }
+          }
+        }
       }
     },
     "node_templates": {
@@ -52,15 +63,8 @@
                 "implementation": {
                   "primary": "component-script",
                   "dependencies": [
-                    "ncclient"
+                    "pyaml"
                   ]
-                },
-                "inputs": {
-                  "endpoint-selector": "default"
-                },
-                "outputs": {
-                  "response-data": "",
-                  "status": "success"
                 }
               }
             }

diff --git a/components/model-catalog/definition-type/starter-type/node_type/component-remote-python-executor.json b/components/model-catalog/definition-type/starter-type/node_type/component-remote-python-executor.json
index 25ada66..9c3c90b 100644 (file)
--- a/components/model-catalog/definition-type/starter-type/node_type/component-remote-python-executor.json
+++ b/components/model-catalog/definition-type/starter-type/node_type/component-remote-python-executor.json
@@ -1,6 +1,12 @@
 {
   "description": "This is Remote Python Execution Component.",
   "version": "1.0.0",
+  "attributes": {
+    "execution-logs": {
+      "required": true,
+      "type": "string"
+    }
+  },
   "capabilities": {
     "component-node": {
       "type": "tosca.capabilities.Node"
@@ -15,25 +21,13 @@
               "description": "Remote Container or Server selector name.",
               "required": false,
               "type": "string",
-              "default": "default"
+              "default": "remote-python"
             },
             "dynamic-properties": {
               "description": "Dynamic Json Content or DSL Json reference.",
               "required": false,
               "type": "json"
             }
-          },
-          "outputs": {
-            "response-data": {
-              "description": "Execution Response Data in JSON format.",
-              "required": false,
-              "type": "json"
-            },
-            "status": {
-              "description": "Status of the Component Execution ( success or failure )",
-              "required": true,
-              "type": "string"
-            }
           }
         }
       }

diff --git a/ms/blueprintsprocessor/application/src/main/resources/application.properties b/ms/blueprintsprocessor/application/src/main/resources/application.properties
index 57bfdd5..e5b1fe5 100755 (executable)
--- a/ms/blueprintsprocessor/application/src/main/resources/application.properties
+++ b/ms/blueprintsprocessor/application/src/main/resources/application.properties
@@ -13,11 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-#logging.level.web=DEBUG
-
 # Web server config
-server.port=8080
-
 blueprintsprocessor.grpcEnable=false
 blueprintsprocessor.httpPort=8080
 blueprintsprocessor.grpcPort=9111
@@ -25,7 +21,8 @@ blueprintsprocessor.grpcPort=9111
 # Blueprint Processor File Execution and Handling Properties
 blueprintsprocessor.blueprintDeployPath=/opt/app/onap/blueprints/deploy
 blueprintsprocessor.blueprintArchivePath=/opt/app/onap/blueprints/archive
-blueprintsprocessor.blueprintWorkingPath=/opt/app/onap/blueprints/work
+blueprintsprocessor.blueprintWorkingPath=/opt/app/onap/blueprints/working
+
 # Primary Database Configuration
 blueprintsprocessor.db.primary.url=jdbc:mysql://db:3306/sdnctl
 blueprintsprocessor.db.primary.username=sdnctl
@@ -54,4 +51,9 @@ blueprintsprocessor.restclient.sdncodl.password=Kp8bJ4SXszM0WXlhak3eHlcse2gAw84v
 blueprintprocessor.resourceResolution.enabled=true
 blueprintprocessor.netconfExecutor.enabled=true
 blueprintprocessor.restConfExecutor.enabled=true
-blueprintprocessor.remoteScriptCommand.enabled=false
\ No newline at end of file
+blueprintprocessor.remoteScriptCommand.enabled=true
+
+blueprintsprocessor.grpcclient.remote-python.type=token-auth
+blueprintsprocessor.grpcclient.remote-python.host=localhost
+blueprintsprocessor.grpcclient.remote-python.port=50051
+blueprintsprocessor.grpcclient.remote-python.token=Basic Y2NzZGthcHBzOmNjc2RrYXBwcw==
\ No newline at end of file

diff --git a/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt b/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt
index feadfbb..a243f44 100644 (file)
--- a/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt
+++ b/ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt
@@ -20,16 +20,20 @@ import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.*
 import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.AbstractComponentFunction
 import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.ExecutionServiceConstant
 import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.RemoteScriptExecutionService
+import org.onap.ccsdk.cds.controllerblueprints.command.api.ResponseStatus
 import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintProcessorException
+import org.onap.ccsdk.cds.controllerblueprints.core.asJsonNode
 import org.onap.ccsdk.cds.controllerblueprints.core.checkFileExists
 import org.onap.ccsdk.cds.controllerblueprints.core.checkNotBlank
 import org.onap.ccsdk.cds.controllerblueprints.core.data.OperationAssignment
 import org.onap.ccsdk.cds.controllerblueprints.core.normalizedFile
+import org.onap.ccsdk.cds.controllerblueprints.core.utils.JacksonUtils
 import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.config.ConfigurableBeanFactory
 import org.springframework.boot.autoconfigure.condition.ConditionalOnBean
 import org.springframework.context.annotation.Scope
 import org.springframework.stereotype.Component
+import java.lang.Exception
 
 @ConditionalOnBean(name = [ExecutionServiceConstant.SERVICE_GRPC_REMOTE_SCRIPT_EXECUTION])
 @Component("component-remote-python-executor")
@@ -53,12 +57,13 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
         val blueprintVersion = bluePrintContext.version()
 
         val operationAssignment: OperationAssignment = bluePrintContext
-                .nodeTemplateInterfaceOperation(nodeTemplateName, interfaceName, operationName)
+            .nodeTemplateInterfaceOperation(nodeTemplateName, interfaceName, operationName)
 
         val artifactName: String = operationAssignment.implementation?.primary
-                ?: throw BluePrintProcessorException("missing primary field to get artifact name for node template ($nodeTemplateName)")
+            ?: throw BluePrintProcessorException("missing primary field to get artifact name for node template ($nodeTemplateName)")
 
-        val artifactDefinition = bluePrintRuntimeService.resolveNodeTemplateArtifactDefinition(nodeTemplateName, artifactName)
+        val artifactDefinition =
+            bluePrintRuntimeService.resolveNodeTemplateArtifactDefinition(nodeTemplateName, artifactName)
 
         checkNotBlank(artifactDefinition.file) { "couldn't get python script path($artifactName)" }
 
@@ -70,7 +75,7 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
         val dynamicProperties = getOperationInput(INPUT_DYNAMIC_PROPERTIES)
 
         // TODO("Python execution command and Resolve some expressions with dynamic properties")
-        val scriptCommand = "${pythonScript.absolutePath}"
+        val scriptCommand = pythonScript.absolutePath
 
         val dependencies = operationAssignment.implementation?.dependencies
 
@@ -78,34 +83,50 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
             // Open GRPC Connection
             remoteScriptExecutionService.init(endPointSelector.asText())
 
+            var executionLogs = ""
+
             // If dependencies are defined, then install in remote server
             if (dependencies != null && dependencies.isNotEmpty()) {
                 val prepareEnvInput = PrepareRemoteEnvInput(requestId = processId,
-                        remoteScriptType = RemoteScriptType.PYTHON,
-                        packages = dependencies
+                    remoteIdentifier = RemoteIdentifier(blueprintName = blueprintName,
+                        blueprintVersion = blueprintVersion),
+                    remoteScriptType = RemoteScriptType.PYTHON,
+                    packages = dependencies
                 )
                 val prepareEnvOutput = remoteScriptExecutionService.prepareEnv(prepareEnvInput)
-                checkNotNull(prepareEnvOutput.status) {
+                executionLogs = prepareEnvOutput.response
+                setOutput(executionLogs)
+                check(prepareEnvOutput.status == StatusType.SUCCESS) {
                     "failed to get prepare remote env response status for requestId(${prepareEnvInput.requestId})"
                 }
             }
 
             val remoteExecutionInput = RemoteScriptExecutionInput(
-                    requestId = processId,
-                    remoteIdentifier = RemoteIdentifier(blueprintName = blueprintName, blueprintVersion = blueprintVersion),
-                    remoteScriptType = RemoteScriptType.PYTHON,
-                    command = scriptCommand)
+                requestId = processId,
+                remoteIdentifier = RemoteIdentifier(blueprintName = blueprintName, blueprintVersion = blueprintVersion),
+                remoteScriptType = RemoteScriptType.PYTHON,
+                command = scriptCommand)
             val remoteExecutionOutput = remoteScriptExecutionService.executeCommand(remoteExecutionInput)
-            checkNotNull(remoteExecutionOutput.status) {
+            executionLogs += remoteExecutionOutput.response
+            setOutput(executionLogs)
+            check(remoteExecutionOutput.status == StatusType.SUCCESS) {
                 "failed to get prepare remote command response status for requestId(${remoteExecutionOutput.requestId})"
             }
+
+        } catch (e: Exception) {
+            log.error("", e)
         } finally {
             remoteScriptExecutionService.close()
         }
     }
 
+    private fun setOutput(executionLogs: String) {
+        bluePrintRuntimeService.setNodeTemplateAttributeValue(nodeTemplateName,
+            "execution-logs", JacksonUtils.jsonNodeFromObject(executionLogs))
+    }
+
     override suspend fun recoverNB(runtimeException: RuntimeException, executionRequest: ExecutionServiceInput) {
         bluePrintRuntimeService.getBluePrintError()
-                .addError("Failed in ComponentJythonExecutor : ${runtimeException.message}")
+            .addError("Failed in ComponentJythonExecutor : ${runtimeException.message}")
     }
 }
\ No newline at end of file

diff --git a/ms/blueprintsprocessor/modules/services/execution-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/execution/RemoteScriptExecutionService.kt b/ms/blueprintsprocessor/modules/services/execution-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/execution/RemoteScriptExecutionService.kt
index 46969b1..7db5f52 100644 (file)
--- a/ms/blueprintsprocessor/modules/services/execution-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/execution/RemoteScriptExecutionService.kt
+++ b/ms/blueprintsprocessor/modules/services/execution-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/execution/RemoteScriptExecutionService.kt
@@ -48,8 +48,8 @@ class GrpcRemoteScriptExecutionService(private val bluePrintGrpcLibPropertyServi
 
     private val log = LoggerFactory.getLogger(GrpcRemoteScriptExecutionService::class.java)!!
 
-    lateinit var channel: ManagedChannel
-    lateinit var commandExecutorServiceGrpc: CommandExecutorServiceGrpc.CommandExecutorServiceFutureStub
+    private var channel: ManagedChannel? = null
+    private lateinit var commandExecutorServiceGrpc: CommandExecutorServiceGrpc.CommandExecutorServiceFutureStub
 
     override suspend fun init(selector: String) {
         // Get the GRPC Client Service based on selector
@@ -94,10 +94,7 @@ class GrpcRemoteScriptExecutionService(private val bluePrintGrpcLibPropertyServi
     }
 
     override suspend fun close() {
-        // TODO('Verify the correct way to close the client conncetion")
-        if (channel != null) {
-            channel.shutdownNow()
-        }
+        channel?.shutdownNow()
     }
 
 
@@ -105,7 +102,7 @@ class GrpcRemoteScriptExecutionService(private val bluePrintGrpcLibPropertyServi
         val correlationId = this.correlationId ?: this.requestId
 
         return PrepareEnvInput.newBuilder()
-            .setIdentifiers(this.remoteIdentifier.asGrpcData())
+            .setIdentifiers(this.remoteIdentifier!!.asGrpcData())
             .setRequestId(this.requestId)
             .setCorrelationId(correlationId)
             .setScriptType(ScriptType.valueOf(this.remoteScriptType.name))
@@ -120,7 +117,7 @@ class GrpcRemoteScriptExecutionService(private val bluePrintGrpcLibPropertyServi
         return ExecutionInput.newBuilder()
             .setRequestId(this.requestId)
             .setCorrelationId(correlationId)
-            .setIdentifiers(this.remoteIdentifier.asGrpcData())
+            .setIdentifiers(this.remoteIdentifier!!.asGrpcData())
             .setScriptType(ScriptType.valueOf(this.remoteScriptType.name))
             .setCommand(this.command)
             .setTimeOut(this.timeOut.toInt())
@@ -129,15 +126,11 @@ class GrpcRemoteScriptExecutionService(private val bluePrintGrpcLibPropertyServi
             .build()
     }
 
-    fun RemoteIdentifier?.asGrpcData(): Identifiers? {
-        return if (this != null) {
-            Identifiers.newBuilder()
-                .setBlueprintName(this.blueprintName)
-                .setBlueprintVersion(this.blueprintVersion)
-                .build()
-        } else {
-            null
-        }
+    fun RemoteIdentifier.asGrpcData(): Identifiers? {
+        return Identifiers.newBuilder()
+            .setBlueprintName(this.blueprintName)
+            .setBlueprintVersion(this.blueprintVersion)
+            .build()
     }
 
     fun Map<String, JsonNode>.asGrpcData(): Struct {

diff --git a/ms/command-executor/src/main/docker/Dockerfile b/ms/command-executor/src/main/docker/Dockerfile
index cac2b95..1137de1 100644 (file)
--- a/ms/command-executor/src/main/docker/Dockerfile
+++ b/ms/command-executor/src/main/docker/Dockerfile
@@ -5,10 +5,8 @@ RUN python -m pip install --upgrade pip
 RUN pip install grpcio==${GRPC_PYTHON_VERSION} grpcio-tools==${GRPC_PYTHON_VERSION}
 RUN pip install virtualenv
 
-RUN mkdir -p /opt/onap/blueprints && mkdir -p /opt/onap/app/python && chmod -R 777 /opt/onap
-
-COPY start.sh /opt/onap/app/start.sh
-RUN chmod u+x /opt/onap/app/start.sh
+COPY start.sh /opt/app/onap/start.sh
+RUN chmod u+x /opt/app/onap/start.sh
 
 COPY @project.build.finalName@-@assembly.id@.tar.gz /source.tar.gz
 RUN tar -xzf /source.tar.gz -C /tmp \
@@ -16,4 +14,7 @@ RUN tar -xzf /source.tar.gz -C /tmp \
  && rm -rf /source.tar.gz \
  && rm -rf /tmp/@project.build.finalName@
 
-ENTRYPOINT /opt/onap/app/start.sh
\ No newline at end of file
+
+VOLUME /opt/app/onap/blueprints/deploy/
+
+ENTRYPOINT /opt/app/onap/start.sh
\ No newline at end of file

diff --git a/ms/command-executor/src/main/docker/distribution.xml b/ms/command-executor/src/main/docker/distribution.xml
index e4ac2f8..7b8c270 100755 (executable)
--- a/ms/command-executor/src/main/docker/distribution.xml
+++ b/ms/command-executor/src/main/docker/distribution.xml
@@ -25,7 +25,7 @@
     <fileSets>
         <fileSet>
             <directory>${project.basedir}/src/main/python</directory>
-            <outputDirectory>/opt/onap/app/python</outputDirectory>
+            <outputDirectory>/opt/app/onap/python</outputDirectory>
         </fileSet>
     </fileSets>
 </assembly>
\ No newline at end of file

diff --git a/ms/command-executor/src/main/docker/start.sh b/ms/command-executor/src/main/docker/start.sh
index 6c0aa00..0dbd7e8 100755 (executable)
--- a/ms/command-executor/src/main/docker/start.sh
+++ b/ms/command-executor/src/main/docker/start.sh
@@ -28,5 +28,5 @@ then
   export BASIC_AUTH="Basic Y2NzZGthcHBzOmNjc2RrYXBwcw=="
 fi
 
-cd /opt/onap/app/python/
+cd /opt/app/onap/python/
 python command_executor_server.py ${APP_PORT} ${BASIC_AUTH}
\ No newline at end of file