build/package.py

   1 #! /usr/bin/env python3
   2 # -*- coding: utf-8 -*-
   3
   4 #   COPYRIGHT NOTICE STARTS HERE
   5
   6 #   Copyright 2019 . Samsung Electronics Co., Ltd.
   7 #
   8 #   Licensed under the Apache License, Version 2.0 (the "License");
   9 #   you may not use this file except in compliance with the License.
  10 #   You may obtain a copy of the License at
  11 #
  12 #       http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 #   Unless required by applicable law or agreed to in writing, software
  15 #   distributed under the License is distributed on an "AS IS" BASIS,
  16 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17 #   See the License for the specific language governing permissions and
  18 #   limitations under the License.
  19
  20 #   COPYRIGHT NOTICE ENDS HERE
  21
  22 from datetime import datetime
  23 import subprocess
  24 import argparse
  25 import logging
  26 import shutil
  27 import glob
  28 import json
  29 import sys
  30 import os
  31
  32 import tarfile
  33 import git
  34
  35 log = logging.getLogger(__name__)
  36 script_location = os.path.dirname(os.path.realpath(__file__))
  37
  38
  39 def prepare_application_repository(directory, url, refspec, patch_path):
  40     """
  41     Downloads git repository according to refspec, applies patch if provided
  42     :param directory: path to repository
  43     :param url: url to repository
  44     :param refspec: refspec to fetch
  45     :param patch_path: path git patch to be applied over repository
  46     :return: repository - git repository object
  47     """
  48
  49     try:
  50         shutil.rmtree(directory)
  51     except FileNotFoundError:
  52         pass
  53
  54     log.info('Cloning {} with refspec {} '.format(url, refspec))
  55     repository = git.Repo.init(directory)
  56     origin = repository.create_remote('origin', url)
  57     origin.pull(refspec)
  58     repository.git.submodule('update', '--init')
  59
  60     if patch_path:
  61         log.info('Applying {} over {} {}'.format(patch_path,
  62                                                  url,
  63                                                  refspec))
  64         repository.git.apply(patch_path)
  65     else:
  66         log.info('No patch file provided, skipping patching')
  67
  68     return repository
  69
  70
  71 def create_package_info_file(output_file, repository_list, tag):
  72     """
  73     Generates text file in json format containing basic information about the build
  74     :param output_file:
  75     :param repository_list: list of repositories to be included in package info
  76     :return:
  77     """
  78     log.info('Generating package.info file')
  79     build_info = {
  80         'Build_info': {
  81             'build_date': datetime.now().strftime('%Y-%m-%d_%H-%M'),
  82             'Version': tag
  83         }
  84     }
  85     for repository in repository_list:
  86         build_info['Build_info'][
  87             repository.config_reader().get_value('remote "origin"', 'url')] = repository.head.commit.hexsha
  88
  89     with open(output_file, 'w') as outfile:
  90         json.dump(build_info, outfile, indent=4)
  91
  92
  93 def create_package(tar_content, file_name):
  94     """
  95     Creates packages
  96     :param tar_content: list of dictionaries defining src file and destination tar file
  97     :param file_name: output file
  98     """
  99     log.info('Creating package {}'.format(file_name))
 100     with tarfile.open(file_name, 'w') as output_tar_file:
 101         for src, dst in tar_content.items():
 102             if src != '':
 103               output_tar_file.add(src, dst)
 104
 105
 106 def build_offline_deliverables(build_version,
 107                                application_repository_url,
 108                                application_repository_reference,
 109                                application_patch_file,
 110                                application_charts_dir,
 111                                application_configuration,
 112                                application_patch_role,
 113                                output_dir,
 114                                resources_directory,
 115                                aux_directory,
 116                                skip_sw,
 117                                skip_resources,
 118                                skip_aux,
 119                                overwrite):
 120     """
 121     Prepares offline deliverables
 122     :param build_version: Version for packages tagging
 123     :param application_repository_url: git repository hosting application helm charts
 124     :param application_repository_reference: git refspec for repository hosting application helm charts
 125     :param application_patch_file: git patch file to be applied over application repository
 126     :param application_charts_dir: path to directory under application repository containing helm charts
 127     :param application_configuration:  path to application configuration file (helm override configuration)
 128     :param application_patch_role: path to application patch role (executed just before helm deploy)
 129     :param output_dir: Destination directory for saving packages
 130     :param resources_directory: Path to resource directory
 131     :param aux_directory: Path to aux binary directory
 132     :param skip_sw: skip sw package generation
 133     :param skip_resources: skip resources package generation
 134     :param skip_aux: skip aux package generation
 135     :param overwrite: overwrite files in output directory
 136     :return:
 137     """
 138
 139     if os.path.exists(output_dir) and os.listdir(output_dir):
 140         if not overwrite:
 141             log.error('Output directory is not empty, use overwrite to force build')
 142             raise FileExistsError
 143         shutil.rmtree(output_dir)
 144
 145     # Git
 146     offline_repository_dir = os.path.join(script_location, '..')
 147     offline_repository = git.Repo(offline_repository_dir)
 148
 149     application_dir = os.path.join(output_dir, 'application_repository')
 150     application_repository = prepare_application_repository(application_dir,
 151                                                             application_repository_url,
 152                                                             application_repository_reference,
 153                                                             application_patch_file)
 154
 155     # Package info
 156     info_file = os.path.join(output_dir, 'package.info')
 157     create_package_info_file(info_file, [application_repository, offline_repository], build_version)
 158
 159     # packages layout as dictionaries. <file> : <file location under tar archive>
 160     sw_content = {
 161         os.path.join(offline_repository_dir, 'ansible'): 'ansible',
 162         application_configuration: 'ansible/application/application_configuration.yml',
 163         application_patch_role: 'ansible/application/onap-patch-role',
 164         os.path.join(application_dir, application_charts_dir): 'ansible/application/helm_charts',
 165         info_file: 'package.info'
 166     }
 167     resources_content = {
 168         resources_directory: '',
 169         info_file: 'package.info'
 170     }
 171     aux_content = {
 172         aux_directory: '',
 173         info_file: 'package.info'
 174     }
 175
 176     if not skip_sw:
 177         log.info('Building offline installer')
 178         os.chdir(os.path.join(offline_repository_dir, 'ansible', 'docker'))
 179         installer_build = subprocess.run(
 180             os.path.join(offline_repository_dir, 'ansible', 'docker', 'build_ansible_image.sh'))
 181         installer_build.check_returncode()
 182         os.chdir(script_location)
 183         sw_package_tar_path = os.path.join(output_dir, 'sw_package' + build_version + '.tar')
 184         create_package(sw_content, sw_package_tar_path)
 185
 186     if not skip_resources:
 187         log.info('Building own dns image')
 188         dns_build = subprocess.run([
 189             os.path.join(offline_repository_dir, 'build', 'creating_data', 'create_nginx_image', '01create-image.sh'),
 190             os.path.join(resources_directory, 'offline_data', 'docker_images_infra')])
 191         dns_build.check_returncode()
 192
 193         # Workaround for downloading without "flat" option
 194         log.info('Binaries - workaround')
 195         download_dir_path = os.path.join(resources_directory, 'downloads')
 196         os.chdir(download_dir_path)
 197         for file in os.listdir():
 198             if os.path.islink(file):
 199                 os.unlink(file)
 200
 201         rke_files = glob.glob(os.path.join('.', '**/rke_linux-amd64'), recursive=True)
 202         os.symlink(rke_files[0], os.path.join(download_dir_path, rke_files[0].split('/')[-1]))
 203
 204         helm_tar_files = glob.glob(os.path.join('.', '**/helm-*-linux-amd64.tar.gz'), recursive=True)
 205         os.symlink(helm_tar_files[0], os.path.join(download_dir_path, helm_tar_files[0].split('/')[-1]))
 206
 207         kubectl_files = glob.glob(os.path.join('.', '**/kubectl'), recursive=True)
 208         os.symlink(kubectl_files[0], os.path.join(download_dir_path, kubectl_files[0].split('/')[-1]))
 209
 210         os.chdir(script_location)
 211         # End of workaround
 212
 213         log.info('Create rhel repo')
 214         createrepo = subprocess.run(['createrepo', os.path.join(resources_directory, 'pkg', 'rhel')])
 215         createrepo.check_returncode()
 216
 217         resources_package_tar_path = os.path.join(output_dir, 'resources_package' + build_version + '.tar')
 218         create_package(resources_content, resources_package_tar_path)
 219
 220     if not skip_aux:
 221         aux_package_tar_path = os.path.join(output_dir, 'aux_package'+ build_version + '.tar')
 222         create_package(aux_content, aux_package_tar_path)
 223
 224     shutil.rmtree(application_dir)
 225
 226
 227 def run_cli():
 228     """
 229     Run as cli tool
 230     """
 231     parser = argparse.ArgumentParser(description='Create Package For Offline Installer')
 232     parser.add_argument('--build-version',
 233                         help='version of the build', default='custom')
 234     parser.add_argument('application_repository_url', metavar='application-repository-url',
 235                         help='git repository hosting application helm charts')
 236     parser.add_argument('--application-repository_reference', default='master',
 237                         help='git refspec for repository hosting application helm charts')
 238     parser.add_argument('--application-patch_file',
 239                         help='git patch file to be applied over application repository', default='')
 240     parser.add_argument('--application-charts_dir',
 241                         help='path to directory under application repository containing helm charts ', default='kubernetes')
 242     parser.add_argument('--application-configuration',
 243                         help='path to application configuration file (helm override configuration)',
 244                         default='')
 245     parser.add_argument('--application-patch-role',
 246                         help='path to application patch role file (ansible role) to be executed right before installation',
 247                         default='')
 248     parser.add_argument('--output-dir', '-o', default=os.path.join(script_location, '..', '..'),
 249                         help='Destination directory for saving packages')
 250     parser.add_argument('--resources-directory', default='',
 251                         help='Path to resource directory')
 252     parser.add_argument('--aux-directory',
 253                         help='Path to aux binary directory', default='')
 254     parser.add_argument('--skip-sw', action='store_true', default=False,
 255                         help='Set to skip sw package generation')
 256     parser.add_argument('--skip-resources', action='store_true', default=False,
 257                         help='Set to skip resources package generation')
 258     parser.add_argument('--skip-aux', action='store_true', default=False,
 259                         help='Set to skip aux package generation')
 260     parser.add_argument('--overwrite', action='store_true', default=False,
 261                         help='overwrite files in output directory')
 262     parser.add_argument('--debug', action='store_true', default=False,
 263                         help='Turn on debug output')
 264     args = parser.parse_args()
 265
 266     if args.debug:
 267         logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
 268     else:
 269         logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s')
 270
 271     build_offline_deliverables(args.build_version,
 272                                args.application_repository_url,
 273                                args.application_repository_reference,
 274                                args.application_patch_file,
 275                                args.application_charts_dir,
 276                                args.application_configuration,
 277                                args.application_patch_role,
 278                                args.output_dir,
 279                                args.resources_directory,
 280                                args.aux_directory,
 281                                args.skip_sw,
 282                                args.skip_resources,
 283                                args.skip_aux,
 284                                args.overwrite)
 285
 286
 287 if __name__ == '__main__':
 288     run_cli()