ms/command-executor/src/main/python/command_executor_handler.py

   1 #
   2 # Copyright (C) 2019 Bell Canada.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #      http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 #
  16 from builtins import Exception, open, dict
  17 from subprocess import CalledProcessError, PIPE
  18
  19 import logging
  20 import os
  21 import subprocess
  22 import sys
  23 import virtualenv
  24 import venv
  25 import utils
  26 import proto.CommandExecutor_pb2 as CommandExecutor_pb2
  27
  28 REQUIREMENTS_TXT = "requirements.txt"
  29
  30
  31 class CommandExecutorHandler():
  32
  33     def __init__(self, request):
  34         self.request = request
  35         self.logger = logging.getLogger(self.__class__.__name__)
  36         self.blueprint_id = utils.get_blueprint_id(request)
  37         self.venv_home = '/opt/app/onap/blueprints/deploy/' + self.blueprint_id
  38         self.installed = self.venv_home + '/.installed'
  39
  40     def is_installed(self):
  41         return os.path.exists(self.installed)
  42
  43     def prepare_env(self, request, results):
  44         if not self.is_installed():
  45             self.create_venv()
  46             if not self.activate_venv():
  47                 return False
  48
  49             f = open(self.installed, "w+")
  50             if not self.install_packages(request, CommandExecutor_pb2.pip, f, results):
  51                 return False
  52             f.write("\r\n")
  53             results.append("\n")
  54             if not self.install_packages(request, CommandExecutor_pb2.ansible_galaxy, f, results):
  55                 return False
  56             f.close()
  57         else:
  58             f = open(self.installed, "r")
  59             results.append(f.read())
  60             f.close()
  61
  62         # deactivate_venv(blueprint_id)
  63         return True
  64
  65     def execute_command(self, request, results):
  66
  67         if not self.activate_venv():
  68             return False
  69
  70         cmd = "cd " + self.venv_home
  71
  72         if "ansible-playbook" in request.command:
  73             cmd = cmd + "; " + request.command + " -e 'ansible_python_interpreter=" + self.venv_home + "/bin/python'"
  74         else:
  75             cmd = cmd + "; " + request.command
  76
  77         self.logger.info("Command: {}".format(cmd))
  78         try:
  79             with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
  80                                   shell=True, bufsize=1, universal_newlines=True) as newProcess:
  81                 while True:
  82                     output = newProcess.stdout.readline()
  83                     if output == '' and newProcess.poll() is not None:
  84                         break
  85                     if output:
  86                         self.logger.info(output.strip())
  87                         results.append(output.strip())
  88                     rc = newProcess.poll()
  89         except Exception as e:
  90             self.logger.info("{} - Failed to execute command. Error: {}".format(self.blueprint_id, e))
  91             results.append(e)
  92             return False
  93
  94         # deactivate_venv(blueprint_id)
  95         return True
  96
  97     def install_packages(self, request, type, f, results):
  98         for package in request.packages:
  99             if package.type == type:
 100                 f.write("Installed %s packages:\r\n" % CommandExecutor_pb2.PackageType.Name(type))
 101                 for p in package.package:
 102                     f.write("   %s\r\n" % p)
 103                     if package.type == CommandExecutor_pb2.pip:
 104                         success = self.install_python_packages(p, results)
 105                     else:
 106                         success = self.install_ansible_packages(p, results)
 107                     if not success:
 108                         f.close()
 109                         os.remove(self.installed)
 110                         return False
 111         return True
 112
 113     def install_python_packages(self, package, results):
 114         self.logger.info(
 115             "{} - Install Python package({}) in Python Virtual Environment".format(self.blueprint_id, package))
 116
 117         if REQUIREMENTS_TXT == package:
 118             command = ["pip", "install", "-r", self.venv_home + "/Environments/" + REQUIREMENTS_TXT]
 119         else:
 120             command = ["pip", "install", package]
 121
 122         env = dict(os.environ)
 123         if "https_proxy" in os.environ:
 124             env['https_proxy'] = os.environ['https_proxy']
 125
 126         try:
 127             results.append(subprocess.run(command, check=True, stdout=PIPE, stderr=PIPE, env=env).stdout.decode())
 128             results.append("\n")
 129             return True
 130         except CalledProcessError as e:
 131             results.append(e.stderr.decode())
 132             return False
 133
 134     def install_ansible_packages(self, package, results):
 135         self.logger.info(
 136             "{} - Install Ansible Role package({}) in Python Virtual Environment".format(self.blueprint_id, package))
 137         command = ["ansible-galaxy", "install", package, "-p", self.venv_home + "/Scripts/ansible/roles"]
 138
 139         env = dict(os.environ)
 140         if "http_proxy" in os.environ:
 141             # ansible galaxy uses https_proxy environment variable, but requires it to be set with http proxy value.
 142             env['https_proxy'] = os.environ['http_proxy']
 143
 144         try:
 145             results.append(subprocess.run(command, check=True, stdout=PIPE, stderr=PIPE, env=env).stdout.decode())
 146             results.append("\n")
 147             return True
 148         except CalledProcessError as e:
 149             results.append(e.stderr.decode())
 150             return False
 151
 152     def create_venv(self):
 153         self.logger.info("{} - Create Python Virtual Environment".format(self.blueprint_id))
 154         try:
 155             bin_dir = self.venv_home + "/bin"
 156             # venv doesn't populate the activate_this.py script, hence we use from virtualenv
 157             venv.create(self.venv_home, with_pip=True, system_site_packages=True)
 158             virtualenv.writefile(os.path.join(bin_dir, "activate_this.py"), virtualenv.ACTIVATE_THIS)
 159         except Exception as err:
 160             self.logger.info(
 161                 "{} - Failed to provision Python Virtual Environment. Error: {}".format(self.blueprint_id, err))
 162
 163     def activate_venv(self):
 164         self.logger.info("{} - Activate Python Virtual Environment".format(self.blueprint_id))
 165
 166         # Fix: The python generated activate_this.py script concatenates the env bin dir to PATH on every call
 167         #      eventually this process PATH variable was so big (128Kb) that no child process could be spawn
 168         #      This script will remove all duplicates; while keeping the order of the PATH folders
 169         fixpathenvvar = "os.environ['PATH']=os.pathsep.join(list(dict.fromkeys(os.environ['PATH'].split(':'))))"
 170
 171         path = "%s/bin/activate_this.py" % self.venv_home
 172         try:
 173             exec (open(path).read(), {'__file__': path})
 174             exec (fixpathenvvar)
 175             self.logger.info("Running with PATH : {}".format(os.environ['PATH']))
 176             return True
 177         except Exception as err:
 178             self.logger.info(
 179                 "{} - Failed to activate Python Virtual Environment. Error: {}".format(self.blueprint_id, err))
 180             return False
 181
 182     def deactivate_venv(self):
 183         self.logger.info("{} - Deactivate Python Virtual Environment".format(self.blueprint_id))
 184         command = ["deactivate"]
 185         try:
 186             subprocess.run(command, check=True)
 187         except Exception as err:
 188             self.logger.info(
 189                 "{} - Failed to deactivate Python Virtual Environment. Error: {}".format(self.blueprint_id, err))