build/download/npm_downloader.py

   1 #! /usr/bin/env python3
   2 # -*- coding: utf-8 -*-
   3
   4 #   COPYRIGHT NOTICE STARTS HERE
   5
   6 #   Copyright 2019 © Samsung Electronics Co., Ltd.
   7 #
   8 #   Licensed under the Apache License, Version 2.0 (the "License");
   9 #   you may not use this file except in compliance with the License.
  10 #   You may obtain a copy of the License at
  11 #
  12 #       http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 #   Unless required by applicable law or agreed to in writing, software
  15 #   distributed under the License is distributed on an "AS IS" BASIS,
  16 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17 #   See the License for the specific language governing permissions and
  18 #   limitations under the License.
  19
  20 #   COPYRIGHT NOTICE ENDS HERE
  21
  22 import argparse
  23 import datetime
  24 import hashlib
  25 import logging
  26 import os
  27 import sys
  28 import timeit
  29
  30 from retrying import retry
  31
  32 import http_downloader
  33 import http_file
  34
  35 log = logging.getLogger(__name__)
  36
  37
  38 class NpmDownloader(http_downloader.HttpDownloader):
  39     def __init__(self, npm_registry, *list_args, workers=None):
  40         super().__init__(*list_args, list_type='npm packages', workers=workers)
  41         self._registry = npm_registry
  42
  43     def _download_item(self, item):
  44         """
  45         Download npm package
  46         :param item: http file to be downloaded (tuple: (npm_name@version, dst_dir))
  47         """
  48         log.info('Downloading: {}'.format(item[0]))
  49         npm_name, npm_version = item[0].split('@')
  50         dst_path = '{}/{}-{}.tgz'.format(item[1], npm_name, npm_version)
  51         try:
  52             tarball = http_file.HttpFile(item[0], self._get_npm(*item[0].split('@')), dst_path)
  53             tarball.save_to_file()
  54         except Exception as err:
  55             log.exception('Failed: {}'.format(item[0]))
  56             if os.path.isfile(dst_path):
  57                 os.remove(dst_path)
  58             raise err
  59         log.info('Downloaded: {}'.format(item[0]))
  60
  61     @retry(stop_max_attempt_number=5, wait_fixed=5000)
  62     def _get_npm(self, npm_name, npm_version):
  63         """
  64         Download and save npm tarball to disk
  65         :param npm_name: name of npm package
  66         :param npm_version: version of npm package
  67         """
  68         npm_url = '{}/{}/{}'.format(self._registry, npm_name, npm_version)
  69         npm_req = self._make_get_request(npm_url)
  70         npm_json = npm_req.json()
  71         tarball_url = npm_json['dist']['tarball']
  72         shasum = npm_json['dist']['shasum']
  73         tarball_req = self._make_get_request(tarball_url)
  74         tarball = tarball_req.content
  75         if hashlib.sha1(tarball).hexdigest() == shasum:
  76             return tarball
  77         else:
  78             raise Exception(
  79                 '{}@{}: Wrong checksum. Retrying...'.format(npm_name, npm_version))
  80
  81     def _is_missing(self, item):
  82         """
  83         Check if item is missing (not downloaded)
  84         :param item: item to check
  85         :return: boolean
  86         """
  87         return not os.path.isfile('{}/{}-{}.tgz'.format(self._data_list[item], *item.split('@')))
  88
  89
  90 def run_cli():
  91     """
  92     Run as cli tool
  93     """
  94     parser = argparse.ArgumentParser(description='Download npm packages from list')
  95     parser.add_argument('npm_list', metavar='npm-list',
  96                         help='File with list of npm packages to download.')
  97     parser.add_argument('--registry', '-r', default='https://registry.npmjs.org',
  98                         help='Download destination')
  99     parser.add_argument('--output-dir', '-o', default=os.getcwd(),
 100                         help='Download destination')
 101     parser.add_argument('--check', '-c', action='store_true', default=False,
 102                         help='Check what is missing. No download.')
 103     parser.add_argument('--debug', action='store_true', default=False,
 104                         help='Turn on debug output')
 105     parser.add_argument('--workers', type=int, default=None,
 106                         help='Set maximum workers for parallel download (default: cores * 5)')
 107
 108     args = parser.parse_args()
 109
 110     if args.debug:
 111         logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
 112     else:
 113         logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s')
 114
 115     downloader = NpmDownloader(args.registry, [args.npm_list, args.output_dir], workers=args.workers)
 116
 117     if args.check:
 118         log.info('Check mode. No download will be executed.')
 119         log.info(downloader.check_table)
 120         sys.exit(0)
 121
 122     timer_start = timeit.default_timer()
 123     try:
 124         downloader.download()
 125     except RuntimeError:
 126         log.error('Error occurred.')
 127         sys.exit(1)
 128     finally:
 129         log.info('Downloading finished in {}'.format(
 130             datetime.timedelta(seconds=timeit.default_timer() - timer_start)))
 131
 132
 133 if __name__ == '__main__':
 134     run_cli()