build/package.py

   1 #! /usr/bin/env python3
   2 # -*- coding: utf-8 -*-
   3
   4 #   COPYRIGHT NOTICE STARTS HERE
   5
   6 #   Copyright 2019 . Samsung Electronics Co., Ltd.
   7 #
   8 #   Licensed under the Apache License, Version 2.0 (the "License");
   9 #   you may not use this file except in compliance with the License.
  10 #   You may obtain a copy of the License at
  11 #
  12 #       http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 #   Unless required by applicable law or agreed to in writing, software
  15 #   distributed under the License is distributed on an "AS IS" BASIS,
  16 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17 #   See the License for the specific language governing permissions and
  18 #   limitations under the License.
  19
  20 #   COPYRIGHT NOTICE ENDS HERE
  21
  22 from datetime import datetime
  23 import subprocess
  24 import argparse
  25 import logging
  26 import shutil
  27 import glob
  28 import json
  29 import sys
  30 import os
  31
  32 import tarfile
  33 import git
  34
  35 log = logging.getLogger(__name__)
  36 script_location = os.path.abspath(os.path.join(__file__, '..'))
  37 offline_repository_dir = os.path.abspath(os.path.join(script_location, '..'))
  38
  39
  40 def prepare_application_repository(directory, url, refspec, patch_path):
  41     """
  42     Downloads git repository according to refspec, applies patch if provided
  43     :param directory: path to repository
  44     :param url: url to repository
  45     :param refspec: refspec to fetch
  46     :param patch_path: path git patch to be applied over repository
  47     :return: repository - git repository object
  48     """
  49
  50     try:
  51         shutil.rmtree(directory)
  52     except FileNotFoundError:
  53         pass
  54
  55     log.info('Cloning {} with refspec {} '.format(url, refspec))
  56     repository = git.Repo.init(directory)
  57     origin = repository.create_remote('origin', url)
  58     origin.pull(refspec)
  59     repository.git.submodule('update', '--init')
  60
  61     if patch_path:
  62         log.info('Applying {} over {} {}'.format(patch_path,
  63                                                  url,
  64                                                  refspec))
  65         repository.git.apply(patch_path)
  66     else:
  67         log.info('No patch file provided, skipping patching')
  68
  69     return repository
  70
  71
  72 def create_package_info_file(output_file, repository_list, tag, metadata):
  73     """
  74     Generates text file in json format containing basic information about the build
  75     :param output_file:
  76     :param repository_list: list of repositories to be included in package info
  77     :param tag: build version of packages
  78     :param metadata: additional metadata into package.info
  79     :return:
  80     """
  81     log.info('Generating package.info file')
  82     build_info = {
  83         'Build_info': {
  84             'build_date': datetime.now().strftime('%Y-%m-%d_%H-%M'),
  85             'Version': tag
  86         }
  87     }
  88     for repository in repository_list:
  89         build_info['Build_info'][
  90             repository.config_reader().get_value('remote "origin"', 'url')] = repository.head.commit.hexsha
  91
  92     if len(metadata) != 0:
  93         build_info['Build_info'][metadata[0]] = metadata[1]
  94
  95     with open(output_file, 'w') as outfile:
  96         json.dump(build_info, outfile, indent=4)
  97
  98
  99 def create_package(tar_content, file_name):
 100     """
 101     Creates packages
 102     :param tar_content: list of dictionaries defining src file and destination tar file
 103     :param file_name: output file
 104     """
 105     log.info('Creating package {}'.format(file_name))
 106     with tarfile.open(file_name, 'w') as output_tar_file:
 107         for src, dst in tar_content.items():
 108             if src != '':
 109                 output_tar_file.add(src, dst)
 110
 111
 112 def build_offline_deliverables(build_version,
 113                                application_repository_url,
 114                                application_repository_reference,
 115                                application_patch_file,
 116                                application_charts_dir,
 117                                application_configuration,
 118                                application_patch_role,
 119                                output_dir,
 120                                resources_directory,
 121                                aux_directory,
 122                                skip_sw,
 123                                skip_resources,
 124                                skip_aux,
 125                                overwrite,
 126                                metadata):
 127     """
 128     Prepares offline deliverables
 129     :param build_version: Version for packages tagging
 130     :param application_repository_url: git repository hosting application helm charts
 131     :param application_repository_reference: git refspec for repository hosting application helm charts
 132     :param application_patch_file: git patch file to be applied over application repository
 133     :param application_charts_dir: path to directory under application repository containing helm charts
 134     :param application_configuration:  path to application configuration file (helm override configuration)
 135     :param application_patch_role: path to application patch role (executed just before helm deploy)
 136     :param output_dir: Destination directory for saving packages
 137     :param resources_directory: Path to resource directory
 138     :param aux_directory: Path to aux binary directory
 139     :param skip_sw: skip sw package generation
 140     :param skip_resources: skip resources package generation
 141     :param skip_aux: skip aux package generation
 142     :param overwrite: overwrite files in output directory
 143     :param metadata: add metadata info into package.info
 144     :return:
 145     """
 146
 147     if os.path.exists(output_dir) and os.listdir(output_dir):
 148         if not overwrite:
 149             log.error('Output directory is not empty, use overwrite to force build')
 150             raise FileExistsError(output_dir)
 151         shutil.rmtree(output_dir)
 152
 153     # Git
 154     offline_repository = git.Repo(offline_repository_dir)
 155
 156     application_dir = os.path.join(output_dir, 'application_repository')
 157     application_repository = prepare_application_repository(application_dir,
 158                                                             application_repository_url,
 159                                                             application_repository_reference,
 160                                                             application_patch_file)
 161
 162     # Package info
 163     info_file = os.path.join(output_dir, 'package.info')
 164     create_package_info_file(info_file, [application_repository, offline_repository], build_version, metadata)
 165
 166     # packages layout as dictionaries. <file> : <file location under tar archive>
 167     sw_content = {
 168         os.path.join(offline_repository_dir, 'ansible'): 'ansible',
 169         application_configuration: 'ansible/application/application_configuration.yml',
 170         application_patch_role: 'ansible/application/onap-patch-role',
 171         os.path.join(application_dir, application_charts_dir): 'ansible/application/helm_charts',
 172         info_file: 'package.info'
 173     }
 174     resources_content = {
 175         resources_directory: '',
 176         info_file: 'package.info'
 177     }
 178     aux_content = {
 179         aux_directory: '',
 180         info_file: 'package.info'
 181     }
 182
 183     if not skip_sw:
 184         log.info('Building offline installer')
 185         os.chdir(os.path.join(offline_repository_dir, 'ansible', 'docker'))
 186         installer_build = subprocess.run(
 187             os.path.join(offline_repository_dir, 'ansible', 'docker', 'build_ansible_image.sh'))
 188         installer_build.check_returncode()
 189         os.chdir(script_location)
 190         sw_package_tar_path = os.path.join(output_dir, 'sw_package.tar')
 191         create_package(sw_content, sw_package_tar_path)
 192
 193     if not skip_resources:
 194         log.info('Building own dns image')
 195         dns_build = subprocess.run([
 196             os.path.join(offline_repository_dir, 'build', 'creating_data', 'create_nginx_image', '01create-image.sh'),
 197             os.path.join(resources_directory, 'offline_data', 'docker_images_infra')])
 198         dns_build.check_returncode()
 199
 200         # Workaround for downloading without "flat" option
 201         log.info('Binaries - workaround')
 202         download_dir_path = os.path.join(resources_directory, 'downloads')
 203         os.chdir(download_dir_path)
 204         for file in os.listdir(download_dir_path):
 205             if os.path.islink(file):
 206                 os.unlink(file)
 207
 208         rke_files = glob.glob(os.path.join('.', '**/rke_linux-amd64'), recursive=True)
 209         os.symlink(rke_files[0], os.path.join(download_dir_path, rke_files[0].split('/')[-1]))
 210
 211         helm_tar_files = glob.glob(os.path.join('.', '**/helm-*-linux-amd64.tar.gz'), recursive=True)
 212         os.symlink(helm_tar_files[0], os.path.join(download_dir_path, helm_tar_files[0].split('/')[-1]))
 213
 214         kubectl_files = glob.glob(os.path.join('.', '**/kubectl'), recursive=True)
 215         os.symlink(kubectl_files[0], os.path.join(download_dir_path, kubectl_files[0].split('/')[-1]))
 216
 217         os.chdir(script_location)
 218         # End of workaround
 219
 220         resources_package_tar_path = os.path.join(output_dir, 'resources_package.tar')
 221         create_package(resources_content, resources_package_tar_path)
 222
 223     if not skip_aux:
 224         aux_package_tar_path = os.path.join(output_dir, 'aux_package.tar')
 225         create_package(aux_content, aux_package_tar_path)
 226
 227     shutil.rmtree(application_dir)
 228
 229
 230 def run_cli():
 231     """
 232     Run as cli tool
 233     """
 234     parser = argparse.ArgumentParser(description='Create Package For Offline Installer')
 235     parser.add_argument('--build-version',
 236                         help='version of the build', default='')
 237     parser.add_argument('application_repository_url', metavar='application-repository-url',
 238                         help='git repository hosting application helm charts')
 239     parser.add_argument('--application-repository_reference', default='master',
 240                         help='git refspec for repository hosting application helm charts')
 241     parser.add_argument('--application-patch_file',
 242                         help='git patch file to be applied over application repository', default='')
 243     parser.add_argument('--application-charts_dir',
 244                         help='path to directory under application repository containing helm charts ',
 245                         default='kubernetes')
 246     parser.add_argument('--application-configuration',
 247                         help='path to application configuration file (helm override configuration)',
 248                         default=os.path.join(offline_repository_dir, 'config/application_configuration.yml'))
 249     parser.add_argument('--application-patch-role',
 250                         help='path to application patch role file (ansible role) to be executed right before installation',
 251                         default='')
 252     parser.add_argument('--output-dir', '-o', default=os.path.join(offline_repository_dir, '../packages'),
 253                         help='Destination directory for saving packages')
 254     parser.add_argument('--resources-directory', default=os.path.join(offline_repository_dir, '../resources'),
 255                         help='Path to resource directory')
 256     parser.add_argument('--aux-directory',
 257                         help='Path to aux binary directory', default='')
 258     parser.add_argument('--skip-sw', action='store_true', default=False,
 259                         help='Set to skip sw package generation')
 260     parser.add_argument('--skip-resources', action='store_true', default=False,
 261                         help='Set to skip resources package generation')
 262     parser.add_argument('--skip-aux', action='store_true', default=False,
 263                         help='Set to skip aux package generation')
 264     parser.add_argument('--overwrite', action='store_true', default=False,
 265                         help='overwrite files in output directory')
 266     parser.add_argument('--debug', action='store_true', default=False,
 267                         help='Turn on debug output')
 268     parser.add_argument('--add-metadata', nargs=2,
 269                         help='additional metadata added into package.info, format: key value', default=[])
 270     args = parser.parse_args()
 271
 272     if args.debug:
 273         logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
 274     else:
 275         logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s')
 276
 277     build_offline_deliverables(args.build_version,
 278                                args.application_repository_url,
 279                                args.application_repository_reference,
 280                                args.application_patch_file,
 281                                args.application_charts_dir,
 282                                args.application_configuration,
 283                                args.application_patch_role,
 284                                args.output_dir,
 285                                args.resources_directory,
 286                                args.aux_directory,
 287                                args.skip_sw,
 288                                args.skip_resources,
 289                                args.skip_aux,
 290                                args.overwrite,
 291                                args.add_metadata)
 292
 293
 294 if __name__ == '__main__':
 295     run_cli()