1 #! /usr/bin/env python3
2 # -*- coding: utf-8 -*-
4 # COPYRIGHT NOTICE STARTS HERE
6 # Copyright 2022 © Samsung Electronics Co., Ltd.
8 # Licensed under the Apache License, Version 2.0 (the "License");
9 # you may not use this file except in compliance with the License.
10 # You may obtain a copy of the License at
12 # http://www.apache.org/licenses/LICENSE-2.0
14 # Unless required by applicable law or agreed to in writing, software
15 # distributed under the License is distributed on an "AS IS" BASIS,
16 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 # See the License for the specific language governing permissions and
18 # limitations under the License.
20 # COPYRIGHT NOTICE ENDS HERE
28 import docker_downloader
30 import http_downloader
32 import pypi_downloader
35 log = logging.getLogger(name=__name__)
40 Parse command line arguments
43 parser = argparse.ArgumentParser(description='Download data from lists')
44 list_group = parser.add_argument_group()
45 list_group.add_argument('--docker', action='append', nargs='+', default=[],
46 metavar=('list', 'dir-name'),
47 help='Docker type list. If second argument is specified '
48 'it is treated as directory where images will be saved '
49 'otherwise only pull operation is executed this can\'t '
50 'be mixed between multiple docker list specifications. '
51 'if one of the list does not have directory specified '
52 'all lists are only pulled!!!')
53 list_group.add_argument('--http', action='append', nargs=2, default=[],
54 metavar=('list', 'dir-name'),
55 help='Http type list and directory to save downloaded files')
56 list_group.add_argument('--npm', action='append', nargs=2, default=[],
57 metavar=('list', 'dir-name'),
58 help='npm type list and directory to save downloaded files')
59 list_group.add_argument('--rpm', action='append', nargs=2, default=[],
60 metavar=('list', 'dir-name'),
61 help='rpm type list and directory to save downloaded files')
62 list_group.add_argument('--git', action='append', nargs=2, default=[],
63 metavar=('list', 'dir-name'),
64 help='git repo type list and directory to save downloaded files')
65 list_group.add_argument('--pypi', action='append', nargs=2, default=[],
66 metavar=('list', 'dir-name'),
67 help='pypi packages type list and directory to save downloaded files')
68 parser.add_argument('--npm-registry', default='https://registry.npmjs.org',
69 help='npm registry to use (default: https://registry.npmjs.org)')
70 parser.add_argument('--docker-private-registry-mirror', default=None, metavar='IP:PORT',
71 help='Address of docker mirroring repository that caches images'
72 ' from private registries to get those images from')
73 parser.add_argument('--check', '-c', action='store_true', default=False,
74 help='Check what is missing. No download.')
75 parser.add_argument('--debug', action='store_true', default=False,
76 help='Turn on debug output')
78 args = parser.parse_args()
80 for arg in ('docker', 'npm', 'http', 'rpm', 'git', 'pypi'):
81 if getattr(args, arg):
84 parser.error('One of --docker, --npm, --http, --rpm, --git or --pypi must be specified')
87 def log_start(item_type):
90 :param item_type: type of resources
93 log.info('Starting download of {}.'.format(item_type))
96 def handle_download(downloader, check_mode, errorred_lists, start_time):
98 Handle download of resources
99 :param downloader: downloader to use
100 :param check_mode: run in check mode (boolean)
101 :param errorred_lists: list of data types of failed lists
102 :param start_time: timeit.default_timer() right before download
103 :return: timeit.default_timer() at the end of download
106 print(downloader.check_table)
108 log_start(downloader.list_type)
110 downloader.download()
112 errorred_lists.append(downloader.list_type)
113 return log_time_interval(start_time, downloader.list_type)
116 def handle_command_download(downloader_class, check_mode, errorred_lists, start_time, *args):
118 Handle download of resources where shell command is used
119 :param downloader_class: Class of command_downloader.CommandDownloader to use
120 :param check_mode: run in check mode (boolean)
121 :param errorred_lists: list of data types of failed lists
122 :param start_time: timeit.default_timer() right before download
123 :param args: arguments for downloader class initialization
124 :return: timeit.default_timer() at the end of download
127 downloader = downloader_class(*args)
128 return handle_download(downloader, check_mode, errorred_lists, start_time)
129 except FileNotFoundError as err:
130 classname = type(downloader_class).__name__
131 log.exception('Error initializing: {}: {}'.format(classname, err))
132 return timeit.default_timer()
135 def log_time_interval(start, resource_type=''):
137 Log how long the download took
138 :param start: timeit.default_timer() when interval started
139 :param resource_type: type of data that was downloaded. (empty string for whole download)
140 :return: timeit.default_timer() after logging
142 e_time = datetime.timedelta(seconds=timeit.default_timer() - start)
144 msg = 'Download of {} took {}\n'.format(resource_type, e_time)
146 msg = 'Execution ended. Total elapsed time {}'.format(e_time)
148 return timeit.default_timer()
152 if sys.version_info.major < 3:
153 log.error('Unfortunately Python 2 is not supported for data download.')
157 console_handler = logging.StreamHandler(sys.stdout)
158 console_formatter = logging.Formatter('%(message)s')
159 console_handler.setFormatter(console_formatter)
160 now = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
161 log_file = 'download_data-{}.log'.format(now)
162 file_format = "%(asctime)s: %(filename)s: %(levelname)s: %(message)s"
165 logging.basicConfig(level=logging.DEBUG, filename=log_file, format=file_format)
167 logging.basicConfig(level=logging.INFO, filename=log_file, format=file_format)
168 root_logger = logging.getLogger()
169 root_logger.addHandler(console_handler)
172 timer_start = interval_start = timeit.default_timer()
175 log.info('Check mode. No download will be executed.')
178 save = True if len(list(filter(lambda x: len(x) == 2, args.docker))) == len(args.docker) else False
179 docker = docker_downloader.DockerDownloader(save, *args.docker, mirror=args.docker_private_registry_mirror, workers=3)
180 interval_start = handle_download(docker, args.check, errorred_lists, interval_start)
183 http = http_downloader.HttpDownloader(*args.http)
184 interval_start = handle_download(http, args.check, errorred_lists, interval_start)
187 npm = npm_downloader.NpmDownloader(args.npm_registry, *args.npm)
188 interval_start = handle_download(npm, args.check, errorred_lists, interval_start)
191 interval_start = handle_command_download(rpm_downloader.RpmDownloader, args.check, errorred_lists,
192 interval_start, *args.rpm)
195 interval_start = handle_command_download(git_downloader.GitDownloader, args.check, errorred_lists,
196 interval_start, *args.git)
199 handle_command_download(pypi_downloader.PyPiDownloader, args.check, errorred_lists,
200 interval_start, *args.pypi)
203 log_time_interval(timer_start)
206 log.error('Errors encountered while processing these types:'
207 '\n{}'.format('\n'.join(errorred_lists)))
211 if __name__ == '__main__':