-#! /usr/bin/env python
+#! /usr/bin/env python3
# -*- coding: utf-8 -*-
# COPYRIGHT NOTICE STARTS HERE
-# Copyright 2019 © Samsung Electronics Co., Ltd.
+# Copyright 2022 © Samsung Electronics Co., Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# COPYRIGHT NOTICE ENDS HERE
import argparse
+import datetime
import logging
import sys
-import datetime
import timeit
-import base
-import docker_images
-import git_repos
-import http_files
-import npm_packages
-import pypi_packages
-import rpm_packages
+import docker_downloader
+import git_downloader
+import http_downloader
+import npm_downloader
+import pypi_downloader
+import rpm_downloader
log = logging.getLogger(name=__name__)
+
def parse_args():
- parser=argparse.ArgumentParser(description='Download data from lists')
+ """
+ Parse command line arguments
+ :return: arguments
+ """
+ parser = argparse.ArgumentParser(description='Download data from lists')
list_group = parser.add_argument_group()
list_group.add_argument('--docker', action='append', nargs='+', default=[],
- metavar=('list', 'dir-name'),
- help='Docker type list. If second argument is specified '
- 'it is treated as directory where images will be saved '
- 'otherwise only pull operation is executed')
+ metavar=('list', 'dir-name'),
+ help='Docker type list. If second argument is specified '
+ 'it is treated as directory where images will be saved '
+ 'otherwise only pull operation is executed this can\'t '
+ 'be mixed between multiple docker list specifications. '
+ 'if one of the list does not have directory specified '
+ 'all lists are only pulled!!!')
list_group.add_argument('--http', action='append', nargs=2, default=[],
- metavar=('list', 'dir-name'),
- help='Http type list and directory to save downloaded files')
+ metavar=('list', 'dir-name'),
+ help='Http type list and directory to save downloaded files')
list_group.add_argument('--npm', action='append', nargs=2, default=[],
- metavar=('list', 'dir-name'),
- help='npm type list and directory to save downloaded files')
+ metavar=('list', 'dir-name'),
+ help='npm type list and directory to save downloaded files')
list_group.add_argument('--rpm', action='append', nargs=2, default=[],
- metavar=('list', 'dir-name'),
- help='rpm type list and directory to save downloaded files')
+ metavar=('list', 'dir-name'),
+ help='rpm type list and directory to save downloaded files')
list_group.add_argument('--git', action='append', nargs=2, default=[],
- metavar=('list', 'dir-name'),
- help='git repo type list and directory to save downloaded files')
+ metavar=('list', 'dir-name'),
+ help='git repo type list and directory to save downloaded files')
list_group.add_argument('--pypi', action='append', nargs=2, default=[],
- metavar=('list', 'dir-name'),
- help='pypi packages type list and directory to save downloaded files')
+ metavar=('list', 'dir-name'),
+ help='pypi packages type list and directory to save downloaded files')
parser.add_argument('--npm-registry', default='https://registry.npmjs.org',
help='npm registry to use (default: https://registry.npmjs.org)')
+ parser.add_argument('--docker-private-registry-mirror', default=None, metavar='HOST:PORT',
+ help='Address of docker mirroring repository that caches images'
+ ' from private registries to get those images from')
+ parser.add_argument('--docker-private-registry-exclude', action='append', default=[], metavar='REGISTRY_NAME',
+ help='The name of a private registry to exclude when using --docker-private-registry-mirror.'
+ ' Images that originate from excluded registry will not be'
+ ' pulled from mirroring repository. This option can be used multiple times.')
parser.add_argument('--check', '-c', action='store_true', default=False,
help='Check what is missing. No download.')
parser.add_argument('--debug', action='store_true', default=False,
if getattr(args, arg):
return args
- parser.error('One of --docker, --npm, --http, --rpm, --git must be specified')
+ parser.error('One of --docker, --npm, --http, --rpm, --git or --pypi must be specified')
+
+
+def log_start(item_type):
+ """
+ Log starting message
+ :param item_type: type of resources
+ :return:
+ """
+ log.info('Starting download of {}.'.format(item_type))
+
+
+def handle_download(downloader, check_mode, errorred_lists, start_time):
+ """
+ Handle download of resources
+ :param downloader: downloader to use
+ :param check_mode: run in check mode (boolean)
+ :param errorred_lists: list of data types of failed lists
+ :param start_time: timeit.default_timer() right before download
+ :return: timeit.default_timer() at the end of download
+ """
+ if check_mode:
+ print(downloader.check_table)
+ else:
+ log_start(downloader.list_type)
+ try:
+ downloader.download()
+ except RuntimeError:
+ errorred_lists.append(downloader.list_type)
+ return log_time_interval(start_time, downloader.list_type)
+
+
+def handle_command_download(downloader_class, check_mode, errorred_lists, start_time, *args):
+ """
+ Handle download of resources where shell command is used
+ :param downloader_class: Class of command_downloader.CommandDownloader to use
+ :param check_mode: run in check mode (boolean)
+ :param errorred_lists: list of data types of failed lists
+ :param start_time: timeit.default_timer() right before download
+ :param args: arguments for downloader class initialization
+ :return: timeit.default_timer() at the end of download
+ """
+ try:
+ downloader = downloader_class(*args)
+ return handle_download(downloader, check_mode, errorred_lists, start_time)
+ except FileNotFoundError as err:
+ classname = type(downloader_class).__name__
+ log.exception('Error initializing: {}: {}'.format(classname, err))
+ return timeit.default_timer()
+
+
+def log_time_interval(start, resource_type=''):
+ """
+ Log how long the download took
+ :param start: timeit.default_timer() when interval started
+ :param resource_type: type of data that was downloaded. (empty string for whole download)
+ :return: timeit.default_timer() after logging
+ """
+ e_time = datetime.timedelta(seconds=timeit.default_timer() - start)
+ if resource_type:
+ msg = 'Download of {} took {}\n'.format(resource_type, e_time)
+ else:
+ msg = 'Execution ended. Total elapsed time {}'.format(e_time)
+ log.info(msg)
+ return timeit.default_timer()
def run_cli():
+ if sys.version_info.major < 3:
+ log.error('Unfortunately Python 2 is not supported for data download.')
+ sys.exit(1)
args = parse_args()
console_handler = logging.StreamHandler(sys.stdout)
root_logger = logging.getLogger()
root_logger.addHandler(console_handler)
- list_with_errors = []
- timer_start = timeit.default_timer()
-
- for docker_list in args.docker:
- log.info('Processing {}.'.format(docker_list[0]))
- progress = None if args.check else base.init_progress('docker images')
- save = False
- if len(docker_list) > 1:
- save = True
- else:
- docker_list.append(None)
- try:
- docker_images.download(docker_list[0], save,
- docker_list[1], args.check, progress)
- except RuntimeError:
- list_with_errors.append(docker_list[0])
+ errorred_lists = []
+ timer_start = interval_start = timeit.default_timer()
- for http_list in args.http:
- progress = None if args.check else base.init_progress('http files')
- log.info('Processing {}.'.format(http_list[0]))
- try:
- http_files.download(http_list[0], http_list[1], args.check,
- progress)
- except RuntimeError:
- list_with_errors.append(http_list[0])
+ if args.check:
+ log.info('Check mode. No download will be executed.')
- for npm_list in args.npm:
- progress = None if args.check else base.init_progress('npm packages')
- log.info('Processing {}.'.format(npm_list[0]))
- try:
- npm_packages.download(npm_list[0], args.npm_registry, npm_list[1],
- args.check, progress)
- except RuntimeError:
- list_with_errors.append(npm_list[0])
+ if args.docker:
+ save = True if len(list(filter(lambda x: len(x) == 2, args.docker))) == len(args.docker) else False
+ docker = docker_downloader.DockerDownloader(save, *args.docker, mirror=args.docker_private_registry_mirror, mirror_exclude=args.docker_private_registry_exclude, workers=3)
+ interval_start = handle_download(docker, args.check, errorred_lists, interval_start)
- for rpm_list in args.rpm:
- if args.check:
- log.info('Check mode for rpm packages is not implemented')
- break
- log.info('Processing {}.'.format(rpm_list[0]))
- try:
- rpm_packages.download(rpm_list[0], rpm_list[1])
- except RuntimeError:
- list_with_errors.append(rpm_list[0])
-
- for git_list in args.git:
- if args.check:
- log.info('Check mode for git repositories is not implemented')
- break
- progress = None if args.check else base.init_progress('git repositories')
- log.info('Processing {}.'.format(git_list[0]))
- try:
- git_repos.download(git_list[0], git_list[1], progress)
- except RuntimeError:
- list_with_errors.append(git_list[0])
-
- for pypi_list in args.pypi:
- if args.check:
- log.info('Check mode for pypi packages is not implemented')
- break
- progress = None if args.check else base.init_progress('pypi packages')
- log.info('Processing {}.'.format(pypi_list[0]))
- try:
- pypi_packages.download(pypi_list[0], pypi_list[1], progress)
- except RuntimeError:
- list_with_errors.append(pypi_list[0])
+ if args.http:
+ http = http_downloader.HttpDownloader(*args.http)
+ interval_start = handle_download(http, args.check, errorred_lists, interval_start)
- e_time = datetime.timedelta(seconds=timeit.default_timer() - timer_start)
- log.info(timeit.default_timer() - timer_start)
- log.info('Execution ended. Total elapsed time {}'.format(e_time))
+ if args.npm:
+ npm = npm_downloader.NpmDownloader(args.npm_registry, *args.npm)
+ interval_start = handle_download(npm, args.check, errorred_lists, interval_start)
- if list_with_errors:
- log.error('Errors encountered while processing these lists:'
- '\n{}'.format('\n'.join(list_with_errors)))
- sys.exit(1)
+ if args.rpm:
+ interval_start = handle_command_download(rpm_downloader.RpmDownloader, args.check, errorred_lists,
+ interval_start, *args.rpm)
+
+ if args.git:
+ interval_start = handle_command_download(git_downloader.GitDownloader, args.check, errorred_lists,
+ interval_start, *args.git)
+ if args.pypi:
+ handle_command_download(pypi_downloader.PyPiDownloader, args.check, errorred_lists,
+ interval_start, *args.pypi)
+
+ if not args.check:
+ log_time_interval(timer_start)
+
+ if errorred_lists:
+ log.error('Errors encountered while processing these types:'
+ '\n{}'.format('\n'.join(errorred_lists)))
+ sys.exit(1)
if __name__ == '__main__':
Code Review / oom / offline-installer.git / blobdiff