build/package.py

   1 #! /usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 #   COPYRIGHT NOTICE STARTS HERE
   5
   6 #   Copyright 2019 . Samsung Electronics Co., Ltd.
   7 #
   8 #   Licensed under the Apache License, Version 2.0 (the "License");
   9 #   you may not use this file except in compliance with the License.
  10 #   You may obtain a copy of the License at
  11 #
  12 #       http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 #   Unless required by applicable law or agreed to in writing, software
  15 #   distributed under the License is distributed on an "AS IS" BASIS,
  16 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17 #   See the License for the specific language governing permissions and
  18 #   limitations under the License.
  19
  20 #   COPYRIGHT NOTICE ENDS HERE
  21
  22 from datetime import datetime
  23 import subprocess
  24 import argparse
  25 import logging
  26 import shutil
  27 import glob
  28 import json
  29 import sys
  30 import os
  31
  32 import tarfile
  33 import git
  34
  35 log = logging.getLogger(__name__)
  36 script_location = os.path.dirname(os.path.realpath(__file__))
  37
  38
  39 def prepare_application_repository(directory, url, refspec, patch_path):
  40     """
  41     Downloads git repository according to refspec, applies patch if provided
  42     :param directory: path to repository
  43     :param url: url to repository
  44     :param refspec: refspec to fetch
  45     :param patch_path: path git patch to be applied over repository
  46     :return: repository - git repository object
  47     """
  48
  49     try:
  50         shutil.rmtree(directory)
  51     except FileNotFoundError:
  52         pass
  53
  54     log.info('Cloning {} with refspec {} '.format(url, refspec))
  55     repository = git.Repo.init(directory)
  56     origin = repository.create_remote('origin', url)
  57     origin.pull(refspec)
  58     repository.git.submodule('update', '--init')
  59
  60     if patch_path:
  61         log.info('Applying {} over {} {}'.format(patch_path,
  62                                                  url,
  63                                                  refspec))
  64         repository.git.apply(patch_path)
  65     else:
  66         log.info('No patch file provided, skipping patching')
  67
  68     return repository
  69
  70
  71 def create_package_info_file(output_file, repository_list):
  72     """
  73     Generates text file in json format containing basic information about the build
  74     :param output_file:
  75     :param repository_list: list of repositories to be included in package info
  76     :return:
  77     """
  78     log.info('Generating package.info file')
  79     build_info = {
  80         'Build_info': {
  81             'build_date': datetime.now().strftime('%Y-%m-%d_%H-%M')
  82         }
  83     }
  84     for repository in repository_list:
  85         build_info['Build_info'][
  86             repository.config_reader().get_value('remote "origin"', 'url')] = repository.head.commit.hexsha
  87
  88     with open(output_file, 'w') as outfile:
  89         json.dump(build_info, outfile, indent=4)
  90
  91
  92 def create_package(tar_content, file_name):
  93     """
  94     Creates packages
  95     :param tar_content: list of dictionaries defining src file and destination tar file
  96     :param file_name: output file
  97     """
  98     log.info('Creating package {}'.format(file_name))
  99     with tarfile.open(file_name, 'w') as output_tar_file:
 100         for src, dst in tar_content.items():
 101             output_tar_file.add(src, dst)
 102
 103
 104 def build_offline_deliverables(application_repository_url,
 105                                application_repository_reference,
 106                                application_patch_file,
 107                                output_dir,
 108                                resources_directory,
 109                                skip_sw,
 110                                skip_resources,
 111                                skip_aux,
 112                                overwrite):
 113     """
 114     Prepares offline deliverables
 115     :param application_repository_url: git repository hosting application helm charts
 116     :param application_repository_reference: git refspec for repository hosting application helm charts
 117     :param application_patch_file: git patch file to be applied over application repository
 118     :param output_dir: Destination directory for saving packages
 119     :param resources_directory: Path to resource directory
 120     :param skip_sw: skip sw package generation
 121     :param skip_resources: skip resources package generation
 122     :param skip_aux: skip aux package generation
 123     :param overwrite: overwrite files in output directory
 124     :return:
 125     """
 126
 127     if os.path.exists(output_dir) and os.listdir(output_dir):
 128         if not overwrite:
 129             log.error('Output directory is not empty, use overwrite to force build')
 130             raise FileExistsError
 131
 132     # Git
 133     offline_repository_dir = os.path.join(script_location, '..')
 134     offline_repository = git.Repo(offline_repository_dir)
 135
 136     application_dir = os.path.join(output_dir, 'application_repository')
 137     application_repository = prepare_application_repository(application_dir,
 138                                                             application_repository_url,
 139                                                             application_repository_reference,
 140                                                             application_patch_file)
 141
 142     # Package info
 143     info_file = os.path.join(output_dir, 'package.info')
 144     create_package_info_file(info_file, [application_repository, offline_repository])
 145
 146     # packages layout as dictionaries. <file> : <file location under tar archive>
 147     sw_content = {
 148         os.path.join(offline_repository_dir, 'ansible'): 'ansible',
 149         os.path.join(offline_repository_dir, 'config',
 150                      'application_configuration.yml'): 'ansible/application/application_configuration.yml',
 151         os.path.join(offline_repository_dir, 'patches', 'onap-patch-role'): 'ansible/application/onap-patch-role',
 152         os.path.join(application_dir, 'kubernetes'): 'ansible/application/helm_charts',
 153         info_file: 'packge.info'
 154     }
 155     resources_content = {
 156         resources_directory: '',
 157         info_file: 'packge.info'
 158     }
 159     aux_content = {
 160         info_file: 'packge.info'
 161     }
 162
 163     if not skip_sw:
 164         log.info('Building offline installer')
 165         os.chdir(os.path.join(offline_repository_dir, 'ansible', 'docker'))
 166         installer_build = subprocess.run(
 167             os.path.join(offline_repository_dir, 'ansible', 'docker', 'build_ansible_image.sh'))
 168         installer_build.check_returncode()
 169         os.chdir(script_location)
 170         sw_package_tar_path = os.path.join(output_dir, 'sw_package.tar')
 171         create_package(sw_content, sw_package_tar_path)
 172
 173     if not skip_resources:
 174         log.info('Building own dns image')
 175         dns_build = subprocess.run([
 176             os.path.join(offline_repository_dir, 'build', 'creating_data', 'create_nginx_image', '01create-image.sh'),
 177             os.path.join(resources_directory, 'offline_data', 'docker_images_infra')])
 178         dns_build.check_returncode()
 179
 180         # Workaround for downloading without "flat" option
 181         log.info('Binaries - workaround')
 182         download_dir_path = os.path.join(resources_directory, 'downloads')
 183         os.chdir(download_dir_path)
 184         for file in os.listdir():
 185             if os.path.islink(file):
 186                 os.unlink(file)
 187
 188         rke_files = glob.glob(os.path.join('.', '**/rke_linux-amd64'), recursive=True)
 189         os.symlink(rke_files[0], os.path.join(download_dir_path, rke_files[0].split('/')[-1]))
 190
 191         helm_tar_files = glob.glob(os.path.join('.', '**/helm-*-linux-amd64.tar.gz'), recursive=True)
 192         os.symlink(helm_tar_files[0], os.path.join(download_dir_path, helm_tar_files[0].split('/')[-1]))
 193
 194         kubectl_files = glob.glob(os.path.join('.', '**/kubectl'), recursive=True)
 195         os.symlink(kubectl_files[0], os.path.join(download_dir_path, kubectl_files[0].split('/')[-1]))
 196
 197         os.chdir(script_location)
 198         # End of workaround
 199
 200         log.info('Create rhel repo')
 201         createrepo = subprocess.run(['createrepo', os.path.join(resources_directory, 'pkg', 'rhel')])
 202         createrepo.check_returncode()
 203
 204         resources_package_tar_path = os.path.join(output_dir, 'resources_package.tar')
 205         create_package(resources_content, resources_package_tar_path)
 206
 207     if not skip_aux:
 208         aux_package_tar_path = os.path.join(output_dir, 'aux_package.tar')
 209         create_package(aux_content, aux_package_tar_path)
 210
 211     shutil.rmtree(application_dir)
 212
 213
 214 def run_cli():
 215     """
 216     Run as cli tool
 217     """
 218     parser = argparse.ArgumentParser(description='Create Package For Offline Installer')
 219     parser.add_argument('application_repository_url', metavar='application-repository-url',
 220                         help='git repository hosting application helm charts')
 221     parser.add_argument('--application-repository_reference', default='master',
 222                         help='git refspec for repository hosting application helm charts')
 223     parser.add_argument('--application-patch_file',
 224                         help='git patch file to be applied over application repository', default='')
 225     parser.add_argument('--output-dir', '-o', default=os.path.join(script_location, '..', '..'),
 226                         help='Destination directory for saving packages')
 227     parser.add_argument('--resources-directory',
 228                         help='Path to resource directory')
 229     parser.add_argument('--skip-sw', action='store_true', default=False,
 230                         help='Set to skip sw package generation')
 231     parser.add_argument('--skip-resources', action='store_true', default=False,
 232                         help='Set to skip resources package generation')
 233     parser.add_argument('--skip-aux', action='store_true', default=False,
 234                         help='Set to skip aux package generation')
 235     parser.add_argument('--overwrite', action='store_true', default=False,
 236                         help='overwrite files in output directory')
 237     parser.add_argument('--debug', action='store_true', default=False,
 238                         help='Turn on debug output')
 239     args = parser.parse_args()
 240
 241     if args.debug:
 242         logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
 243     else:
 244         logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s')
 245
 246     build_offline_deliverables(args.application_repository_url,
 247                                args.application_repository_reference,
 248                                args.application_patch_file,
 249                                args.output_dir,
 250                                args.resources_directory,
 251                                args.skip_sw,
 252                                args.skip_resources,
 253                                args.skip_aux,
 254                                args.overwrite)
 255
 256
 257 if __name__ == '__main__':
 258     run_cli()
 259