build/package.py

   1 #! /usr/bin/env python3
   2 # -*- coding: utf-8 -*-
   3
   4 #   COPYRIGHT NOTICE STARTS HERE
   5
   6 #   Copyright 2019 . Samsung Electronics Co., Ltd.
   7 #
   8 #   Licensed under the Apache License, Version 2.0 (the "License");
   9 #   you may not use this file except in compliance with the License.
  10 #   You may obtain a copy of the License at
  11 #
  12 #       http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 #   Unless required by applicable law or agreed to in writing, software
  15 #   distributed under the License is distributed on an "AS IS" BASIS,
  16 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17 #   See the License for the specific language governing permissions and
  18 #   limitations under the License.
  19
  20 #   COPYRIGHT NOTICE ENDS HERE
  21
  22 from datetime import datetime
  23 import subprocess
  24 import argparse
  25 import logging
  26 import shutil
  27 import glob
  28 import json
  29 import sys
  30 import os
  31
  32 import tarfile
  33 import git
  34
  35 log = logging.getLogger(__name__)
  36 script_location = os.path.abspath(os.path.join(__file__, '..'))
  37 offline_repository_dir = os.path.abspath(os.path.join(script_location, '..'))
  38
  39
  40 def prepare_application_repository(directory, url, refspec, patch_path):
  41     """
  42     Downloads git repository according to refspec, applies patch if provided
  43     :param directory: path to repository
  44     :param url: url to repository
  45     :param refspec: refspec to fetch
  46     :param patch_path: path git patch to be applied over repository
  47     :return: repository - git repository object
  48     """
  49
  50     try:
  51         shutil.rmtree(directory)
  52     except FileNotFoundError:
  53         pass
  54
  55     log.info('Cloning {} with refspec {} '.format(url, refspec))
  56     repository = git.Repo.init(directory)
  57     origin = repository.create_remote('origin', url)
  58     origin.pull(refspec)
  59     repository.git.submodule('update', '--init')
  60
  61     if patch_path:
  62         log.info('Applying {} over {} {}'.format(patch_path,
  63                                                  url,
  64                                                  refspec))
  65         repository.git.apply(patch_path)
  66     else:
  67         log.info('No patch file provided, skipping patching')
  68
  69     return repository
  70
  71
  72 def create_package_info_file(output_file, repository_list, tag, metadata):
  73     """
  74     Generates text file in json format containing basic information about the build
  75     :param output_file:
  76     :param repository_list: list of repositories to be included in package info
  77     :param tag: build version of packages
  78     :param metadata: additional metadata into package.info
  79     :return:
  80     """
  81     log.info('Generating package.info file')
  82     build_info = {
  83         'Build_info': {
  84             'build_date': datetime.now().strftime('%Y-%m-%d_%H-%M'),
  85             'Version': tag
  86         }
  87     }
  88     for repository in repository_list:
  89         build_info['Build_info'][
  90             repository.config_reader().get_value('remote "origin"', 'url')] = repository.head.commit.hexsha
  91
  92     if len(metadata) != 0:
  93         build_info['Build_info'][metadata[0]] = metadata[1]
  94
  95     with open(output_file, 'w') as outfile:
  96         json.dump(build_info, outfile, indent=4)
  97
  98
  99 def create_package(tar_content, file_name):
 100     """
 101     Creates packages
 102     :param tar_content: list of dictionaries defining src file and destination tar file
 103     :param file_name: output file
 104     """
 105     log.info('Creating package {}'.format(file_name))
 106     with tarfile.open(file_name, 'w') as output_tar_file:
 107         for src, dst in tar_content.items():
 108             if src != '':
 109                 output_tar_file.add(src, dst)
 110
 111
 112 def build_offline_deliverables(build_version,
 113                                application_repository_url,
 114                                application_repository_reference,
 115                                application_patch_file,
 116                                application_charts_dir,
 117                                application_configuration,
 118                                application_patch_role,
 119                                output_dir,
 120                                resources_directory,
 121                                aux_directory,
 122                                skip_sw,
 123                                skip_resources,
 124                                skip_aux,
 125                                overwrite,
 126                                metadata):
 127     """
 128     Prepares offline deliverables
 129     :param build_version: Version for packages tagging
 130     :param application_repository_url: git repository hosting application helm charts
 131     :param application_repository_reference: git refspec for repository hosting application helm charts
 132     :param application_patch_file: git patch file to be applied over application repository
 133     :param application_charts_dir: path to directory under application repository containing helm charts
 134     :param application_configuration:  path to application configuration file (helm override configuration)
 135     :param application_patch_role: path to application patch role (executed just before helm deploy)
 136     :param output_dir: Destination directory for saving packages
 137     :param resources_directory: Path to resource directory
 138     :param aux_directory: Path to aux binary directory
 139     :param skip_sw: skip sw package generation
 140     :param skip_resources: skip resources package generation
 141     :param skip_aux: skip aux package generation
 142     :param overwrite: overwrite files in output directory
 143     :param metadata: add metadata info into package.info
 144     :return:
 145     """
 146
 147     if os.path.exists(output_dir) and os.listdir(output_dir):
 148         if not overwrite:
 149             log.error('Output directory is not empty, use overwrite to force build')
 150             raise FileExistsError(output_dir)
 151         shutil.rmtree(output_dir)
 152
 153     # Git
 154     offline_repository = git.Repo(offline_repository_dir)
 155
 156     application_dir = os.path.join(output_dir, 'application_repository')
 157     application_repository = prepare_application_repository(application_dir,
 158                                                             application_repository_url,
 159                                                             application_repository_reference,
 160                                                             application_patch_file)
 161
 162     # Package info
 163     info_file = os.path.join(output_dir, 'package.info')
 164     create_package_info_file(info_file, [application_repository, offline_repository], build_version, metadata)
 165
 166     # packages layout as dictionaries. <file> : <file location under tar archive>
 167     sw_content = {
 168         os.path.join(offline_repository_dir, 'ansible'): 'ansible',
 169         application_configuration: 'ansible/application/application_configuration.yml',
 170         application_patch_role: 'ansible/application/onap-patch-role',
 171         os.path.join(application_dir, application_charts_dir): 'ansible/application/helm_charts',
 172         info_file: 'package.info'
 173     }
 174     resources_content = {
 175         resources_directory: '',
 176         info_file: 'package.info'
 177     }
 178     aux_content = {
 179         aux_directory: '',
 180         info_file: 'package.info'
 181     }
 182
 183     # add separator if build version not empty
 184     build_version = "-" + build_version if build_version != "" else ""
 185
 186     if not skip_sw:
 187         log.info('Building offline installer')
 188         os.chdir(os.path.join(offline_repository_dir, 'ansible', 'docker'))
 189         installer_build = subprocess.run(
 190             os.path.join(offline_repository_dir, 'ansible', 'docker', 'build_ansible_image.sh'))
 191         installer_build.check_returncode()
 192         os.chdir(script_location)
 193         sw_package_tar_path = os.path.join(output_dir, 'sw_package' + build_version + '.tar')
 194         create_package(sw_content, sw_package_tar_path)
 195
 196     if not skip_resources:
 197         log.info('Building own dns image')
 198         dns_build = subprocess.run([
 199             os.path.join(offline_repository_dir, 'build', 'creating_data', 'create_nginx_image', '01create-image.sh'),
 200             os.path.join(resources_directory, 'offline_data', 'docker_images_infra')])
 201         dns_build.check_returncode()
 202
 203         # Workaround for downloading without "flat" option
 204         log.info('Binaries - workaround')
 205         download_dir_path = os.path.join(resources_directory, 'downloads')
 206         os.chdir(download_dir_path)
 207         for file in os.listdir(download_dir_path):
 208             if os.path.islink(file):
 209                 os.unlink(file)
 210
 211         rke_files = glob.glob(os.path.join('.', '**/rke_linux-amd64'), recursive=True)
 212         os.symlink(rke_files[0], os.path.join(download_dir_path, rke_files[0].split('/')[-1]))
 213
 214         helm_tar_files = glob.glob(os.path.join('.', '**/helm-*-linux-amd64.tar.gz'), recursive=True)
 215         os.symlink(helm_tar_files[0], os.path.join(download_dir_path, helm_tar_files[0].split('/')[-1]))
 216
 217         kubectl_files = glob.glob(os.path.join('.', '**/kubectl'), recursive=True)
 218         os.symlink(kubectl_files[0], os.path.join(download_dir_path, kubectl_files[0].split('/')[-1]))
 219
 220         os.chdir(script_location)
 221         # End of workaround
 222
 223         resources_package_tar_path = os.path.join(output_dir, 'resources_package' + build_version + '.tar')
 224         create_package(resources_content, resources_package_tar_path)
 225
 226     if not skip_aux:
 227         aux_package_tar_path = os.path.join(output_dir, 'aux_package' + build_version + '.tar')
 228         create_package(aux_content, aux_package_tar_path)
 229
 230     shutil.rmtree(application_dir)
 231
 232
 233 def run_cli():
 234     """
 235     Run as cli tool
 236     """
 237     parser = argparse.ArgumentParser(description='Create Package For Offline Installer')
 238     parser.add_argument('--build-version',
 239                         help='version of the build', default='')
 240     parser.add_argument('application_repository_url', metavar='application-repository-url',
 241                         help='git repository hosting application helm charts')
 242     parser.add_argument('--application-repository_reference', default='master',
 243                         help='git refspec for repository hosting application helm charts')
 244     parser.add_argument('--application-patch_file',
 245                         help='git patch file to be applied over application repository', default='')
 246     parser.add_argument('--application-charts_dir',
 247                         help='path to directory under application repository containing helm charts ',
 248                         default='kubernetes')
 249     parser.add_argument('--application-configuration',
 250                         help='path to application configuration file (helm override configuration)',
 251                         default=os.path.join(offline_repository_dir, 'config/application_configuration.yml'))
 252     parser.add_argument('--application-patch-role',
 253                         help='path to application patch role file (ansible role) to be executed right before installation',
 254                         default='')
 255     parser.add_argument('--output-dir', '-o', default=os.path.join(offline_repository_dir, '../packages'),
 256                         help='Destination directory for saving packages')
 257     parser.add_argument('--resources-directory', default=os.path.join(offline_repository_dir, '../resources'),
 258                         help='Path to resource directory')
 259     parser.add_argument('--aux-directory',
 260                         help='Path to aux binary directory', default='')
 261     parser.add_argument('--skip-sw', action='store_true', default=False,
 262                         help='Set to skip sw package generation')
 263     parser.add_argument('--skip-resources', action='store_true', default=False,
 264                         help='Set to skip resources package generation')
 265     parser.add_argument('--skip-aux', action='store_true', default=False,
 266                         help='Set to skip aux package generation')
 267     parser.add_argument('--overwrite', action='store_true', default=False,
 268                         help='overwrite files in output directory')
 269     parser.add_argument('--debug', action='store_true', default=False,
 270                         help='Turn on debug output')
 271     parser.add_argument('--add-metadata', nargs=2,
 272                         help='additional metadata added into package.info, format: key value', default=[])
 273     args = parser.parse_args()
 274
 275     if args.debug:
 276         logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
 277     else:
 278         logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s')
 279
 280     build_offline_deliverables(args.build_version,
 281                                args.application_repository_url,
 282                                args.application_repository_reference,
 283                                args.application_patch_file,
 284                                args.application_charts_dir,
 285                                args.application_configuration,
 286                                args.application_patch_role,
 287                                args.output_dir,
 288                                args.resources_directory,
 289                                args.aux_directory,
 290                                args.skip_sw,
 291                                args.skip_resources,
 292                                args.skip_aux,
 293                                args.overwrite,
 294                                args.add_metadata)
 295
 296
 297 if __name__ == '__main__':
 298     run_cli()