Update package type name 25/85825/5
authorAlexis de Talhouët <adetalhouet89@gmail.com>
Fri, 19 Apr 2019 13:16:26 +0000 (09:16 -0400)
committerAlexis de Talhouët <adetalhouet89@gmail.com>
Mon, 22 Apr 2019 14:23:09 +0000 (14:23 +0000)
- split log output: prepare-env and exec-cmd
- fix proxy support when not provided

Change-Id: I40cdf90e7e3ebe31b0152923f8b6a38f6b6e9ca0
Issue-ID: CCCSDK-125
Signed-off-by: Alexis de Talhouët <adetalhouet89@gmail.com>
12 files changed:
components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/data_types.json
components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/node_types.json
components/model-catalog/blueprint-model/test-blueprint/remote_scripts/Definitions/remote_scripts.json
components/model-catalog/definition-type/starter-type/data_type/dt-system-packages.json
components/model-catalog/definition-type/starter-type/node_type/component-remote-python-executor.json
components/model-catalog/proto-definition/proto/CommandExecutor.proto
components/model-catalog/proto-definition/proto/README
ms/blueprintsprocessor/functions/python-executor/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/functions/python/executor/ComponentRemotePythonExecutor.kt
ms/blueprintsprocessor/modules/services/execution-service/src/main/kotlin/org/onap/ccsdk/cds/blueprintsprocessor/services/execution/RemoteScriptExecutionService.kt
ms/command-executor/src/main/python/command_executor_handler.py
ms/command-executor/src/main/python/proto/CommandExecutor_pb2.py
ms/command-executor/src/main/python/server.py

index df594ab..25efb36 100644 (file)
@@ -10,7 +10,7 @@
           "constraints": [
             {
               "valid_values": [
-                "ANSIBLE", "PYTHON"
+                "ansible_galaxy", "pip"
               ]
             }
           ]
index 0ee00b3..640a4eb 100644 (file)
@@ -4,8 +4,12 @@
       "description": "This is Remote Python Execution Component.",
       "version": "1.0.0",
       "attributes": {
-        "execution-logs": {
-          "required": true,
+        "prepare-environment-logs": {
+          "required": false,
+          "type": "string"
+        },
+        "execute-command-logs": {
+          "required": false,
           "type": "string"
         }
       },
index 80ef02f..da09e4f 100644 (file)
         },
         "inputs": {},
         "outputs": {
-          "logs": {
-            "type": "json",
+          "prepare-environment-logs": {
+            "type": "string",
             "value": {
               "get_attribute": [
                 "execute-remote-python",
-                "execution-logs"
+                "prepare-environment-logs"
+              ]
+            }
+          },
+          "execute-command-logs": {
+            "type": "string",
+            "value": {
+              "get_attribute": [
+                "execute-remote-python",
+                "execute-command-logs"
               ]
             }
           }
         },
         "inputs": {},
         "outputs": {
-          "logs": {
-            "type": "json",
+          "prepare-environment-logs": {
+            "type": "string",
+            "value": {
+              "get_attribute": [
+                "execute-remote-ansible",
+                "prepare-environment-logs"
+              ]
+            }
+          },
+          "execute-command-logs": {
+            "type": "string",
             "value": {
               "get_attribute": [
                 "execute-remote-ansible",
-                "execution-logs"
+                "execute-command-logs"
               ]
             }
           }
             "operations": {
               "process": {
                 "implementation": {
-                  "primary": "component-script",
-                  "dependencies": [
-                    "pyaml"
-                  ]
+                  "primary": "component-script"
                 },
                 "inputs": {
-                  "command": "python SamplePython.py blah"
+                  "command": "python SamplePython.py blah",
+                  "packages": [
+                    {
+                      "type": "pip",
+                      "package": [
+                        "pyaml"
+                      ]
+                    }
+                  ]
                 }
               }
             }
                   "command": "ansible-playbook first.yaml",
                   "packages": [
                     {
-                      "type": "PYTHON",
+                      "type": "pip",
                       "package": [
                         "ansible"
                       ]
                     },
                     {
-                      "type": "ANSIBLE",
+                      "type": "ansible_galaxy",
                       "package": [
                         "juniper.junos"
                       ]
index 81bb030..e70aa5e 100644 (file)
@@ -8,8 +8,8 @@
       "constraints": [
         {
           "valid_values": [
-            "ANSIBLE",
-            "PYTHON"
+            "ansible_galaxy",
+            "pip"
           ]
         }
       ]
index 3eaee76..5c9503f 100644 (file)
@@ -2,8 +2,12 @@
   "description": "This is Remote Python Execution Component.",
   "version": "1.0.0",
   "attributes": {
-    "execution-logs": {
-      "required": true,
+    "prepare-environment-logs": {
+      "required": false,
+      "type": "string"
+    },
+    "execute-command-logs": {
+      "required": false,
       "type": "string"
     }
   },
index 8f02b8a..bc175db 100644 (file)
@@ -53,8 +53,8 @@ message Packages {
 }
 
 enum PackageType {
-    PYTHON = 0;
-    ANSIBLE = 1;
+    pip = 0;
+    ansible_galaxy = 1;
 }
 
 service CommandExecutorService {
index 47bbf3f..8cc9c48 100644 (file)
@@ -2,11 +2,10 @@ To create python bindings,
 
 Prerequisites:
   https://developers.google.com/protocol-buffers/docs/downloads
-  https://github.com/dropbox/mypy-protobuf
-
+  or
+  https://grpc.io/docs/quickstart/python.html
 
 Command:
-  protoc -I=. --python_out=. --mypy_out=. CommandExecutor.proto
-
-
-
+  protoc -I=. --python_out=. --grpc_python_out=. CommandExecutor.proto
+  or
+  python -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. CommandExecutor.proto
index 17d5fc7..488e0d0 100644 (file)
@@ -22,9 +22,8 @@ import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.*
 import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.AbstractComponentFunction
 import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.ExecutionServiceConstant
 import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.RemoteScriptExecutionService
-import org.onap.ccsdk.cds.controllerblueprints.command.api.ResponseStatus
 import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintProcessorException
-import org.onap.ccsdk.cds.controllerblueprints.core.asJsonNode
+import org.onap.ccsdk.cds.controllerblueprints.core.asJsonPrimitive
 import org.onap.ccsdk.cds.controllerblueprints.core.checkFileExists
 import org.onap.ccsdk.cds.controllerblueprints.core.checkNotBlank
 import org.onap.ccsdk.cds.controllerblueprints.core.data.OperationAssignment
@@ -50,6 +49,9 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
         const val INPUT_DYNAMIC_PROPERTIES = "dynamic-properties"
         const val INPUT_COMMAND = "command"
         const val INPUT_PACKAGES = "packages"
+
+        const val ATTRIBUTE_PREPARE_ENV_LOG = "prepare-environment-logs"
+        const val ATTRIBUTE_EXEC_CMD_LOG = "execute-command-logs"
     }
 
     override suspend fun processNB(executionRequest: ExecutionServiceInput) {
@@ -83,14 +85,10 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
         // TODO("Python execution command and Resolve some expressions with dynamic properties")
         val scriptCommand = command.replace(pythonScript.name, pythonScript.absolutePath)
 
-//        val dependencies = operationAssignment.implementation?.dependencies
-
         try {
             // Open GRPC Connection
             remoteScriptExecutionService.init(endPointSelector.asText())
 
-            var executionLogs = ""
-
             // If packages are defined, then install in remote server
             if (packages !is MissingNode && packages !is NullNode) {
                 val prepareEnvInput = PrepareRemoteEnvInput(requestId = processId,
@@ -99,8 +97,7 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
                     packages = packages
                 )
                 val prepareEnvOutput = remoteScriptExecutionService.prepareEnv(prepareEnvInput)
-                executionLogs = prepareEnvOutput.response
-                setOutput(executionLogs)
+                setAttribute(ATTRIBUTE_PREPARE_ENV_LOG, prepareEnvOutput.response.asJsonPrimitive())
                 check(prepareEnvOutput.status == StatusType.SUCCESS) {
                     "failed to get prepare remote env response status for requestId(${prepareEnvInput.requestId})"
                 }
@@ -111,24 +108,18 @@ open class ComponentRemotePythonExecutor(private val remoteScriptExecutionServic
                 remoteIdentifier = RemoteIdentifier(blueprintName = blueprintName, blueprintVersion = blueprintVersion),
                 command = scriptCommand)
             val remoteExecutionOutput = remoteScriptExecutionService.executeCommand(remoteExecutionInput)
-            executionLogs += remoteExecutionOutput.response
-            setOutput(executionLogs)
+            setAttribute(ATTRIBUTE_EXEC_CMD_LOG, remoteExecutionOutput.response.asJsonPrimitive())
             check(remoteExecutionOutput.status == StatusType.SUCCESS) {
                 "failed to get prepare remote command response status for requestId(${remoteExecutionOutput.requestId})"
             }
 
         } catch (e: Exception) {
-            log.error("", e)
+            log.error("Failed to process on remote executor", e)
         } finally {
             remoteScriptExecutionService.close()
         }
     }
 
-    private fun setOutput(executionLogs: String) {
-        bluePrintRuntimeService.setNodeTemplateAttributeValue(nodeTemplateName,
-            "execution-logs", JacksonUtils.jsonNodeFromObject(executionLogs))
-    }
-
     override suspend fun recoverNB(runtimeException: RuntimeException, executionRequest: ExecutionServiceInput) {
         bluePrintRuntimeService.getBluePrintError()
             .addError("Failed in ComponentJythonExecutor : ${runtimeException.message}")
index 99d4f8c..7aee95e 100644 (file)
@@ -107,7 +107,7 @@ class GrpcRemoteScriptExecutionService(private val bluePrintGrpcLibPropertyServi
             val pckage = Packages.newBuilder()
             JsonFormat.parser().merge(it.toString(), pckage)
             packageList.add(pckage.build())
-            }
+        }
 
         return PrepareEnvInput.newBuilder()
             .setIdentifiers(this.remoteIdentifier!!.asGrpcData())
index 248e443..9eb36fb 100644 (file)
@@ -47,11 +47,11 @@ class CommandExecutorHandler():
                 return False
 
             f = open(self.installed, "w+")
-            if not self.install_packages(request, CommandExecutor_pb2.PYTHON, f, results):
+            if not self.install_packages(request, CommandExecutor_pb2.pip, f, results):
                 return False
             f.write("\r\n")
             results.append("\n")
-            if not self.install_packages(request, CommandExecutor_pb2.ANSIBLE, f, results):
+            if not self.install_packages(request, CommandExecutor_pb2.ansible_galaxy, f, results):
                 return False
             f.close()
         else:
@@ -63,8 +63,8 @@ class CommandExecutorHandler():
         return True
 
     def execute_command(self, request, results):
-        if not self.activate_venv():
-            return False
+        if not self.activate_venv():
+            return False
 
         try:
             results.append(os.popen(request.command).read())
@@ -82,7 +82,7 @@ class CommandExecutorHandler():
                 f.write("Installed %s packages:\r\n" % CommandExecutor_pb2.PackageType.Name(type))
                 for python_package in package.package:
                     f.write("   %s\r\n" % python_package)
-                    if package.type == CommandExecutor_pb2.PYTHON:
+                    if package.type == CommandExecutor_pb2.pip:
                         success = self.install_python_packages(python_package, results)
                     else:
                         success = self.install_ansible_packages(python_package, results)
@@ -98,7 +98,8 @@ class CommandExecutorHandler():
         command = ["pip", "install", package]
 
         env = dict(os.environ)
-        env['https_proxy'] = os.environ['https_proxy']
+        if "https_proxy" in os.environ:
+            env['https_proxy'] = os.environ['https_proxy']
 
         try:
             results.append(subprocess.run(command, check=True, stdout=PIPE, stderr=PIPE, env=env).stdout.decode())
@@ -111,11 +112,12 @@ class CommandExecutorHandler():
     def install_ansible_packages(self, package, results):
         self.logger.info(
             "{} - Install Ansible Role package({}) in Python Virtual Environment".format(self.blueprint_id, package))
-        command = ["ansible-galaxy", "install", package, "-p", "Scripts/ansible/roles"]
+        command = ["ansible-galaxy", "install", package, "-p", self.venv_home + "/Scripts/ansible/roles"]
 
         env = dict(os.environ)
-        # ansible galaxy uses https_proxy environment variable, but requires it to be set with http proxy value.
-        env['https_proxy'] = os.environ['http_proxy']
+        if "http_proxy" in os.environ:
+            # ansible galaxy uses https_proxy environment variable, but requires it to be set with http proxy value.
+            env['https_proxy'] = os.environ['http_proxy']
 
         try:
             results.append(subprocess.run(command, check=True, stdout=PIPE, stderr=PIPE, env=env).stdout.decode())
index 76fc19d..3afeb35 100644 (file)
@@ -22,7 +22,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
   package='org.onap.ccsdk.cds.controllerblueprints.command.api',
   syntax='proto3',
   serialized_options=_b('P\001'),
-  serialized_pb=_b('\n\x15\x43ommandExecutor.proto\x12\x33org.onap.ccsdk.cds.controllerblueprints.command.api\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x8f\x02\n\x0e\x45xecutionInput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x15\n\rcorrelationId\x18\x02 \x01(\t\x12U\n\x0bidentifiers\x18\x03 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x0f\n\x07\x63ommand\x18\x04 \x01(\t\x12\x0f\n\x07timeOut\x18\x05 \x01(\x05\x12+\n\nproperties\x18\x06 \x01(\x0b\x32\x17.google.protobuf.Struct\x12-\n\ttimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xd0\x02\n\x0fPrepareEnvInput\x12U\n\x0bidentifiers\x18\x01 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x11\n\trequestId\x18\x02 \x01(\t\x12\x15\n\rcorrelationId\x18\x03 \x01(\t\x12O\n\x08packages\x18\x04 \x03(\x0b\x32=.org.onap.ccsdk.cds.controllerblueprints.command.api.Packages\x12\x0f\n\x07timeOut\x18\x05 \x01(\x05\x12+\n\nproperties\x18\x06 \x01(\x0b\x32\x17.google.protobuf.Struct\x12-\n\ttimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\">\n\x0bIdentifiers\x12\x15\n\rblueprintName\x18\x01 \x01(\t\x12\x18\n\x10\x62lueprintVersion\x18\x02 \x01(\t\"\xba\x01\n\x0f\x45xecutionOutput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x10\n\x08response\x18\x02 \x01(\t\x12S\n\x06status\x18\x03 \x01(\x0e\x32\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ResponseStatus\x12-\n\ttimestamp\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"k\n\x08Packages\x12N\n\x04type\x18\x01 \x01(\x0e\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.PackageType\x12\x0f\n\x07package\x18\x02 \x03(\t**\n\x0eResponseStatus\x12\x0b\n\x07SUCCESS\x10\x00\x12\x0b\n\x07\x46\x41ILURE\x10\x01*&\n\x0bPackageType\x12\n\n\x06PYTHON\x10\x00\x12\x0b\n\x07\x41NSIBLE\x10\x01\x32\xd1\x02\n\x16\x43ommandExecutorService\x12\x98\x01\n\nprepareEnv\x12\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.PrepareEnvInput\x1a\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutput\x12\x9b\x01\n\x0e\x65xecuteCommand\x12\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionInput\x1a\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutputB\x02P\x01\x62\x06proto3')
+  serialized_pb=_b('\n\x15\x43ommandExecutor.proto\x12\x33org.onap.ccsdk.cds.controllerblueprints.command.api\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x8f\x02\n\x0e\x45xecutionInput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x15\n\rcorrelationId\x18\x02 \x01(\t\x12U\n\x0bidentifiers\x18\x03 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x0f\n\x07\x63ommand\x18\x04 \x01(\t\x12\x0f\n\x07timeOut\x18\x05 \x01(\x05\x12+\n\nproperties\x18\x06 \x01(\x0b\x32\x17.google.protobuf.Struct\x12-\n\ttimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xd0\x02\n\x0fPrepareEnvInput\x12U\n\x0bidentifiers\x18\x01 \x01(\x0b\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.Identifiers\x12\x11\n\trequestId\x18\x02 \x01(\t\x12\x15\n\rcorrelationId\x18\x03 \x01(\t\x12O\n\x08packages\x18\x04 \x03(\x0b\x32=.org.onap.ccsdk.cds.controllerblueprints.command.api.Packages\x12\x0f\n\x07timeOut\x18\x05 \x01(\x05\x12+\n\nproperties\x18\x06 \x01(\x0b\x32\x17.google.protobuf.Struct\x12-\n\ttimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\">\n\x0bIdentifiers\x12\x15\n\rblueprintName\x18\x01 \x01(\t\x12\x18\n\x10\x62lueprintVersion\x18\x02 \x01(\t\"\xba\x01\n\x0f\x45xecutionOutput\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x10\n\x08response\x18\x02 \x01(\t\x12S\n\x06status\x18\x03 \x01(\x0e\x32\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ResponseStatus\x12-\n\ttimestamp\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"k\n\x08Packages\x12N\n\x04type\x18\x01 \x01(\x0e\x32@.org.onap.ccsdk.cds.controllerblueprints.command.api.PackageType\x12\x0f\n\x07package\x18\x02 \x03(\t**\n\x0eResponseStatus\x12\x0b\n\x07SUCCESS\x10\x00\x12\x0b\n\x07\x46\x41ILURE\x10\x01**\n\x0bPackageType\x12\x07\n\x03pip\x10\x00\x12\x12\n\x0e\x61nsible_galaxy\x10\x01\x32\xd1\x02\n\x16\x43ommandExecutorService\x12\x98\x01\n\nprepareEnv\x12\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.PrepareEnvInput\x1a\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutput\x12\x9b\x01\n\x0e\x65xecuteCommand\x12\x43.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionInput\x1a\x44.org.onap.ccsdk.cds.controllerblueprints.command.api.ExecutionOutputB\x02P\x01\x62\x06proto3')
   ,
   dependencies=[google_dot_protobuf_dot_struct__pb2.DESCRIPTOR,google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,])
 
@@ -56,26 +56,26 @@ _PACKAGETYPE = _descriptor.EnumDescriptor(
   file=DESCRIPTOR,
   values=[
     _descriptor.EnumValueDescriptor(
-      name='PYTHON', index=0, number=0,
+      name='pip', index=0, number=0,
       serialized_options=None,
       type=None),
     _descriptor.EnumValueDescriptor(
-      name='ANSIBLE', index=1, number=1,
+      name='ansible_galaxy', index=1, number=1,
       serialized_options=None,
       type=None),
   ],
   containing_type=None,
   serialized_options=None,
   serialized_start=1160,
-  serialized_end=1198,
+  serialized_end=1202,
 )
 _sym_db.RegisterEnumDescriptor(_PACKAGETYPE)
 
 PackageType = enum_type_wrapper.EnumTypeWrapper(_PACKAGETYPE)
 SUCCESS = 0
 FAILURE = 1
-PYTHON = 0
-ANSIBLE = 1
+pip = 0
+ansible_galaxy = 1
 
 
 
@@ -415,8 +415,8 @@ _COMMANDEXECUTORSERVICE = _descriptor.ServiceDescriptor(
   file=DESCRIPTOR,
   index=0,
   serialized_options=None,
-  serialized_start=1201,
-  serialized_end=1538,
+  serialized_start=1205,
+  serialized_end=1542,
   methods=[
   _descriptor.MethodDescriptor(
     name='prepareEnv',