test/mocks/mass-pnf-sim/MassPnfSim.py

   1 #!/usr/bin/env python3
   2 import logging
   3 from subprocess import run, CalledProcessError
   4 import argparse
   5 import ipaddress
   6 from sys import exit
   7 from os import chdir, getcwd, path, popen, kill
   8 from shutil import copytree, rmtree
   9 from json import dumps
  10 from yaml import load, SafeLoader
  11 from glob import glob
  12 from docker import from_env
  13 from requests import get, codes
  14 from requests.exceptions import MissingSchema, InvalidSchema, InvalidURL, ConnectionError, ConnectTimeout
  15
  16 def validate_url(url):
  17     '''Helper function to perform --urlves input param validation'''
  18     logger = logging.getLogger("urllib3")
  19     logger.setLevel(logging.WARNING)
  20     try:
  21         get(url, timeout=0.001)
  22     except (MissingSchema, InvalidSchema, InvalidURL):
  23         raise argparse.ArgumentTypeError(f'{url} is not a valid URL')
  24     except (ConnectionError, ConnectTimeout):
  25         pass
  26     return url
  27
  28 def validate_ip(ip):
  29     '''Helper function to validate input param is a vaild IP address'''
  30     try:
  31         ip_valid = ipaddress.ip_address(ip)
  32     except ValueError:
  33         raise argparse.ArgumentTypeError(f'{ip} is not a valid IP address')
  34     else:
  35         return ip_valid
  36
  37 def get_parser():
  38     '''Process input arguments'''
  39
  40     parser = argparse.ArgumentParser()
  41     subparsers = parser.add_subparsers(title='Subcommands', dest='subcommand')
  42     # Build command parser
  43     subparsers.add_parser('build', help='Build simulator image')
  44     # Bootstrap command parser
  45     parser_bootstrap = subparsers.add_parser('bootstrap', help='Bootstrap the system')
  46     parser_bootstrap.add_argument('--count', help='Instance count to bootstrap', type=int, metavar='INT', default=1)
  47     parser_bootstrap.add_argument('--urlves', help='URL of the VES collector', type=validate_url, metavar='URL', required=True)
  48     parser_bootstrap.add_argument('--ipfileserver', help='Visible IP of the file server (SFTP/FTPS) to be included in the VES event',
  49                                   type=validate_ip, metavar='IP', required=True)
  50     parser_bootstrap.add_argument('--typefileserver', help='Type of the file server (SFTP/FTPS) to be included in the VES event',
  51                                   type=str, choices=['sftp', 'ftps'], required=True)
  52     parser_bootstrap.add_argument('--ipstart', help='IP address range beginning', type=validate_ip, metavar='IP', required=True)
  53     # Start command parser
  54     parser_start = subparsers.add_parser('start', help='Start instances')
  55     parser_start.add_argument('--count', help='Instance count to start', type=int, metavar='INT', default=0)
  56     # Stop command parser
  57     parser_stop = subparsers.add_parser('stop', help='Stop instances')
  58     parser_stop.add_argument('--count', help='Instance count to stop', type=int, metavar='INT', default=0)
  59     # Trigger command parser
  60     parser_trigger = subparsers.add_parser('trigger', help='Trigger one single VES event from each simulator')
  61     parser_trigger.add_argument('--count', help='Instance count to trigger', type=int, metavar='INT', default=0)
  62     # Trigger-custom command parser
  63     parser_triggerstart = subparsers.add_parser('trigger_custom', help='Trigger one single VES event from specific simulators')
  64     parser_triggerstart.add_argument('--triggerstart', help='First simulator id to trigger', type=int,
  65                                      metavar='INT', required=True)
  66     parser_triggerstart.add_argument('--triggerend', help='Last simulator id to trigger', type=int,
  67                                      metavar='INT', required=True)
  68     # Status command parser
  69     parser_status = subparsers.add_parser('status', help='Status')
  70     parser_status.add_argument('--count', help='Instance count to show status for', type=int, metavar='INT', default=0)
  71     # Clean command parser
  72     subparsers.add_parser('clean', help='Clean work-dirs')
  73     # General options parser
  74     parser.add_argument('--verbose', help='Verbosity level', choices=['info', 'debug'],
  75                         type=str, default='info')
  76     return parser
  77
  78 class MassPnfSim:
  79
  80     # MassPnfSim class actions decorator
  81     class _MassPnfSim_Decorators:
  82
  83         @staticmethod
  84         def do_action(action_string, cmd):
  85             def action_decorator(method):
  86                 def action_wrap(self):
  87                     # Alter looping range if action is 'tigger_custom'
  88                     if method.__name__ == 'trigger_custom':
  89                         iter_range = [self.args.triggerstart, self.args.triggerend+1]
  90                     else:
  91                         if not self.args.count:
  92                             # If no instance count set explicitly via --count
  93                             # option
  94                             iter_range = [self.existing_sim_instances]
  95                         else:
  96                             iter_range = [self.args.count]
  97                     method(self)
  98                     for i in range(*iter_range):
  99                         self.logger.info(f'{action_string} {self.sim_dirname_pattern}{i} instance:')
 100                         self._run_cmd(cmd, f"{self.sim_dirname_pattern}{i}")
 101                 return action_wrap
 102             return action_decorator
 103
 104     log_lvl = logging.INFO
 105     sim_config = 'config/config.yml'
 106     sim_port = 5000
 107     sim_base_url = 'http://{}:' + str(sim_port) + '/simulator'
 108     sim_container_name = 'pnf-simulator'
 109     rop_script_name = 'ROP_file_creator.sh'
 110
 111     def __init__(self, args):
 112         self.args = args
 113         self.logger = logging.getLogger(__name__)
 114         self.logger.setLevel(self.log_lvl)
 115         self.sim_dirname_pattern = "pnf-sim-lw-"
 116         self.mvn_build_cmd = 'mvn clean package docker:build -Dcheckstyle.skip'
 117         self.docker_compose_status_cmd = 'docker-compose ps'
 118         self.existing_sim_instances = self._enum_sim_instances()
 119
 120         # Validate 'trigger_custom' subcommand options
 121         if self.args.subcommand == 'trigger_custom':
 122             if (self.args.triggerend + 1) > self.existing_sim_instances:
 123                 self.logger.error('--triggerend value greater than existing instance count.')
 124                 exit(1)
 125
 126         # Validate --count option for subcommands that support it
 127         if self.args.subcommand in ['start', 'stop', 'trigger', 'status']:
 128             if self.args.count > self.existing_sim_instances:
 129                 self.logger.error('--count value greater that existing instance count')
 130                 exit(1)
 131             if not self.existing_sim_instances:
 132                 self.logger.error('No bootstrapped instance found')
 133                 exit(1)
 134
 135         # Validate 'bootstrap' subcommand
 136         if (self.args.subcommand == 'bootstrap') and self.existing_sim_instances:
 137             self.logger.error('Bootstrapped instances detected, not overwiriting, clean first')
 138             exit(1)
 139
 140     def _run_cmd(self, cmd, dir_context='.'):
 141         if self.args.verbose == 'debug':
 142             cmd='bash -x ' + cmd
 143         old_pwd = getcwd()
 144         try:
 145             chdir(dir_context)
 146             run(cmd, check=True, shell=True)
 147             chdir(old_pwd)
 148         except FileNotFoundError:
 149             self.logger.error(f"Directory {dir_context} not found")
 150         except CalledProcessError as e:
 151             exit(e.returncode)
 152
 153     def _enum_sim_instances(self):
 154         '''Helper method that returns bootstraped simulator instances count'''
 155         return len(glob(f"{self.sim_dirname_pattern}[0-9]*"))
 156
 157     def _get_sim_instance_data(self, instance_id):
 158         '''Helper method that returns specific instance data'''
 159         oldpwd = getcwd()
 160         chdir(f"{self.sim_dirname_pattern}{instance_id}")
 161         with open(self.sim_config) as cfg:
 162             yml = load(cfg, Loader=SafeLoader)
 163         chdir(oldpwd)
 164         return yml['ippnfsim']
 165
 166     def _get_docker_containers(self):
 167         '''Returns a list containing 'name' attribute of running docker containers'''
 168         dc = from_env()
 169         containers = []
 170         for container in dc.containers.list():
 171             containers.append(container.attrs['Name'][1:])
 172         return containers
 173
 174     def _get_iter_range(self):
 175         '''Helper routine to get the iteration range
 176         for the lifecycle commands'''
 177         if not self.args.count:
 178             return [self.existing_sim_instances]
 179         else:
 180             return [self.args.count]
 181
 182     def bootstrap(self):
 183         self.logger.info("Bootstrapping PNF instances")
 184
 185         start_port = 2000
 186         ftps_pasv_port_start = 8000
 187         ftps_pasv_port_num_of_ports = 10
 188
 189         ftps_pasv_port_end = ftps_pasv_port_start + ftps_pasv_port_num_of_ports
 190
 191         for i in range(self.args.count):
 192             self.logger.info(f"PNF simulator instance: {i}")
 193
 194             # The IP ranges are in distance of 16 compared to each other.
 195             # This is matching the /28 subnet mask used in the dockerfile inside.
 196             instance_ip_offset = i * 16
 197             ip_properties = [
 198                       'subnet',
 199                       'gw',
 200                       'PnfSim',
 201                       'ftps',
 202                       'sftp'
 203                     ]
 204
 205             ip_offset = 0
 206             ip = {}
 207             for prop in ip_properties:
 208                 ip.update({prop: str(self.args.ipstart + ip_offset + instance_ip_offset)})
 209                 ip_offset += 1
 210
 211             self.logger.debug(f'Instance #{i} properties:\n {dumps(ip, indent=4)}')
 212
 213             PortSftp = start_port + 1
 214             PortFtps = start_port + 2
 215             start_port += 2
 216
 217             self.logger.info(f'\tCreating {self.sim_dirname_pattern}{i}')
 218             copytree('pnf-sim-lightweight', f'{self.sim_dirname_pattern}{i}')
 219
 220             composercmd = " ".join([
 221                     "./simulator.sh compose",
 222                     ip['gw'],
 223                     ip['subnet'],
 224                     str(i),
 225                     self.args.urlves,
 226                     ip['PnfSim'],
 227                     str(self.args.ipfileserver),
 228                     self.args.typefileserver,
 229                     str(PortSftp),
 230                     str(PortFtps),
 231                     ip['ftps'],
 232                     ip['sftp'],
 233                     str(ftps_pasv_port_start),
 234                     str(ftps_pasv_port_end)
 235                 ])
 236             self.logger.debug(f"Script cmdline: {composercmd}")
 237             self.logger.info(f"\tCreating instance #{i} configuration ")
 238             self._run_cmd(composercmd, f"{self.sim_dirname_pattern}{i}")
 239
 240             ftps_pasv_port_start += ftps_pasv_port_num_of_ports + 1
 241             ftps_pasv_port_end += ftps_pasv_port_num_of_ports + 1
 242
 243             self.logger.info(f'Done setting up instance #{i}')
 244
 245     def build(self):
 246         self.logger.info("Building simulator image")
 247         if path.isfile('pnf-sim-lightweight/pom.xml'):
 248             self._run_cmd(self.mvn_build_cmd, 'pnf-sim-lightweight')
 249         else:
 250             self.logger.error('POM file was not found, Maven cannot run')
 251             exit(1)
 252
 253     def clean(self):
 254         self.logger.info('Cleaning simulators workdirs')
 255         for sim_id in range(self.existing_sim_instances):
 256             rmtree(f"{self.sim_dirname_pattern}{sim_id}")
 257
 258     @_MassPnfSim_Decorators.do_action('Starting', './simulator.sh start')
 259     def start(self):
 260         pass
 261
 262     def status(self):
 263         for i in range(*self._get_iter_range()):
 264             self.logger.info(f'Getting {self.sim_dirname_pattern}{i} instance status:')
 265             if f"{self.sim_container_name}-{i}" in self._get_docker_containers():
 266                 try:
 267                     sim_ip = self._get_sim_instance_data(i)
 268                     self.logger.info(f' PNF-Sim IP: {sim_ip}')
 269                     self._run_cmd(self.docker_compose_status_cmd, f"{self.sim_dirname_pattern}{i}")
 270                     sim_response = get('{}/status'.format(self.sim_base_url).format(sim_ip))
 271                     if sim_response.status_code == codes.ok:
 272                         self.logger.info(sim_response.text)
 273                     else:
 274                         self.logger.error(f'Simulator request returned http code {sim_response.status_code}')
 275                 except KeyError:
 276                     self.logger.error(f'Unable to get sim instance IP from {self.sim_config}')
 277             else:
 278                 self.logger.info(' Simulator containers are down')
 279
 280     def stop(self):
 281         for i in range(*self._get_iter_range()):
 282             self.logger.info(f'Stopping {self.sim_dirname_pattern}{i} instance:')
 283             self.logger.info(f' PNF-Sim IP: {self._get_sim_instance_data(i)}')
 284             # attempt killing ROP script
 285             rop_pid = []
 286             for ps_line in iter(popen(f'ps --no-headers -C {self.rop_script_name} -o pid,cmd').readline, ''):
 287                 # try getting ROP script pid
 288                 try:
 289                     ps_line_arr = ps_line.split()
 290                     assert self.rop_script_name in ps_line_arr[2]
 291                     assert ps_line_arr[3] == str(i)
 292                     rop_pid = ps_line_arr[0]
 293                 except AssertionError:
 294                     pass
 295                 else:
 296                     # get rop script childs, kill ROP script and all childs
 297                     childs = popen(f'pgrep -P {rop_pid}').read().split()
 298                     for pid in [rop_pid] + childs:
 299                         kill(int(pid), 15)
 300                     self.logger.info(f' ROP_file_creator.sh {i} successfully killed')
 301             if not rop_pid:
 302                 # no process found
 303                 self.logger.warning(f' ROP_file_creator.sh {i} already not running')
 304             # try tearing down docker-compose application
 305             if f"{self.sim_container_name}-{i}" in self._get_docker_containers():
 306                 self._run_cmd('docker-compose down', self.sim_dirname_pattern + str(i))
 307                 self._run_cmd('docker-compose rm', self.sim_dirname_pattern + str(i))
 308             else:
 309                 self.logger.warning(" Simulator containers are already down")
 310
 311     @_MassPnfSim_Decorators.do_action('Triggering', './simulator.sh trigger-simulator')
 312     def trigger(self):
 313         self.logger.info("Triggering VES sending:")
 314
 315     @_MassPnfSim_Decorators.do_action('Triggering', './simulator.sh trigger-simulator')
 316     def trigger_custom(self):
 317         self.logger.info("Triggering VES sending by a range of simulators:")