build/download/base.py

   1 #! /usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 #   COPYRIGHT NOTICE STARTS HERE
   5
   6 #   Copyright 2019 © Samsung Electronics Co., Ltd.
   7 #
   8 #   Licensed under the Apache License, Version 2.0 (the "License");
   9 #   you may not use this file except in compliance with the License.
  10 #   You may obtain a copy of the License at
  11 #
  12 #       http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 #   Unless required by applicable law or agreed to in writing, software
  15 #   distributed under the License is distributed on an "AS IS" BASIS,
  16 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17 #   See the License for the specific language governing permissions and
  18 #   limitations under the License.
  19
  20 #   COPYRIGHT NOTICE ENDS HERE
  21
  22
  23 import concurrent.futures
  24 import os
  25 import progressbar
  26 import prettytable
  27 import requests
  28 from distutils.spawn import find_executable
  29
  30 progressbar.streams.wrap_stdout()
  31 progressbar.streams.wrap_stderr()
  32
  33
  34 def load_list(item_list):
  35     """
  36     Parse list with items to be downloaded.
  37     :param item_list: File with list of items (1 line per item)
  38     :return: set of items from file
  39     """
  40     with open(item_list, 'r') as f:
  41         return {item for item in (line.strip() for line in f) if item}
  42
  43
  44 def init_progress(items_name):
  45     progress_widgets = ['Downloading {}: '.format(items_name),
  46                         progressbar.Bar(), ' ',
  47                         progressbar.Percentage(), ' ',
  48                         '(', progressbar.SimpleProgress(), ')']
  49
  50     progress = progressbar.ProgressBar(widgets=progress_widgets,
  51                                        poll_rate=1.0,
  52                                        redirect_stdout=True)
  53     return progress
  54
  55
  56 def start_progress(progress, target_count, skipping, log):
  57     log_skipping(skipping, log)
  58     log.info("Initializing download. Takes a while.")
  59
  60     progress.max_value = target_count
  61     progress.start()
  62     progress.update(len(skipping))
  63
  64
  65 def log_skipping(skipping_iterable, logger):
  66     for skipped in skipping_iterable:
  67         logger.info('Skipping: {}'.format(skipped))
  68
  69
  70 def run_concurrent(workers, progress, fn, iterable, *args):
  71     with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
  72         futures = [executor.submit(fn, item, *args) for item in iterable]
  73         error_count = 0
  74         for future in concurrent.futures.as_completed(futures):
  75             error = future.exception()
  76             if error:
  77                 error_count += 1
  78                 progress.update()
  79             else:
  80                 progress.update(progress.value +1)
  81     return error_count
  82
  83
  84 def finish_progress(progress, error_count, log):
  85     progress.finish(dirty=error_count > 0)
  86     log.info('Download ended. Elapsed time {}'.format(progress.data()['time_elapsed']))
  87
  88 def check_tool(name):
  89     return find_executable(name)
  90
  91 def save_to_file(dst, content):
  92     """
  93     Save downloaded byte content to file
  94     :param dst: path to file to save content to
  95     :param content: byte content of file
  96     """
  97     dst_dir = os.path.dirname(dst)
  98     if not os.path.exists(dst_dir):
  99         os.makedirs(dst_dir)
 100     with open(dst, 'wb') as dst_file:
 101         dst_file.write(content)
 102
 103 def make_get_request(url):
 104     req = requests.get(url)
 105     req.raise_for_status()
 106     return req
 107
 108 def simple_check_table(target, missing):
 109     table = prettytable.PrettyTable(['Name', 'Downloaded'])
 110     table.align['Name'] = 'l'
 111     for item in sorted(target):
 112         table.add_row([item, item not in missing])
 113     return table
 114