build/download/base.py

   1 #! /usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 #   COPYRIGHT NOTICE STARTS HERE
   5
   6 #   Copyright 2019 © Samsung Electronics Co., Ltd.
   7 #
   8 #   Licensed under the Apache License, Version 2.0 (the "License");
   9 #   you may not use this file except in compliance with the License.
  10 #   You may obtain a copy of the License at
  11 #
  12 #       http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 #   Unless required by applicable law or agreed to in writing, software
  15 #   distributed under the License is distributed on an "AS IS" BASIS,
  16 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17 #   See the License for the specific language governing permissions and
  18 #   limitations under the License.
  19
  20 #   COPYRIGHT NOTICE ENDS HERE
  21
  22
  23 import progressbar
  24 import concurrent.futures
  25 from distutils.spawn import find_executable
  26
  27 progressbar.streams.wrap_stdout()
  28 progressbar.streams.wrap_stderr()
  29
  30
  31 def load_list(item_list):
  32     """
  33     Parse list with items to be downloaded.
  34     :param item_list: File with list of items (1 line per item)
  35     :return: set of items from file
  36     """
  37     with open(item_list, 'r') as f:
  38         return {item for item in (line.strip() for line in f) if item}
  39
  40
  41 def init_progress(items_name):
  42     progress_widgets = ['Downloading {}: '.format(items_name),
  43                         progressbar.Bar(), ' ',
  44                         progressbar.Percentage(), ' ',
  45                         '(', progressbar.SimpleProgress(), ')']
  46
  47     progress = progressbar.ProgressBar(widgets=progress_widgets,
  48                                        poll_rate=1.0,
  49                                        redirect_stdout=True)
  50     return progress
  51
  52
  53 def start_progress(progress, target_count, skipping, log):
  54     log_skipping(skipping, log)
  55     log.info("Initializing download. Takes a while.")
  56
  57     progress.max_value = target_count
  58     progress.start()
  59     progress.update(len(skipping))
  60
  61
  62 def log_skipping(skipping_iterable, logger):
  63     for skipped in skipping_iterable:
  64         logger.info('Skipping: {}'.format(skipped))
  65
  66
  67 def run_concurrent(workers, progress, fn, iterable, *args):
  68     with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
  69         futures = [executor.submit(fn, item, *args) for item in iterable]
  70         error_count = 0
  71         for future in concurrent.futures.as_completed(futures):
  72             error = future.exception()
  73             if error:
  74                 error_count += 1
  75                 progress.update()
  76             else:
  77                 progress.update(progress.value +1)
  78     return error_count
  79
  80
  81 def finish_progress(progress, error_count, log):
  82     progress.finish(dirty=error_count > 0)
  83     log.info('Download ended. Elapsed time {}'.format(progress.data()['time_elapsed']))
  84
  85 def check_tool(name):
  86     return find_executable(name)