build/package.py

   1 #! /usr/bin/env python3
   2 # -*- coding: utf-8 -*-
   3
   4 #   COPYRIGHT NOTICE STARTS HERE
   5
   6 #   Copyright 2019 . Samsung Electronics Co., Ltd.
   7 #
   8 #   Licensed under the Apache License, Version 2.0 (the "License");
   9 #   you may not use this file except in compliance with the License.
  10 #   You may obtain a copy of the License at
  11 #
  12 #       http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 #   Unless required by applicable law or agreed to in writing, software
  15 #   distributed under the License is distributed on an "AS IS" BASIS,
  16 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17 #   See the License for the specific language governing permissions and
  18 #   limitations under the License.
  19
  20 #   COPYRIGHT NOTICE ENDS HERE
  21
  22 from datetime import datetime
  23 import subprocess
  24 import argparse
  25 import logging
  26 import shutil
  27 import glob
  28 import json
  29 import sys
  30 import os
  31
  32 import tarfile
  33 import git
  34
  35 log = logging.getLogger(__name__)
  36 script_location = os.path.abspath(os.path.join(__file__, '..'))
  37 offline_repository_dir = os.path.abspath(os.path.join(script_location, '..'))
  38
  39
  40 def prepare_application_repository(directory, url, refspec, patch_path):
  41     """
  42     Downloads git repository according to refspec, applies patch if provided
  43     :param directory: path to repository
  44     :param url: url to repository
  45     :param refspec: refspec to fetch
  46     :param patch_path: path git patch to be applied over repository
  47     :return: repository - git repository object
  48     """
  49
  50     try:
  51         shutil.rmtree(directory)
  52     except FileNotFoundError:
  53         pass
  54
  55     log.info('Cloning {} with refspec {} '.format(url, refspec))
  56     repository = git.Repo.init(directory)
  57     origin = repository.create_remote('origin', url)
  58     origin.pull(refspec)
  59     repository.git.submodule('update', '--init')
  60
  61     if patch_path:
  62         log.info('Applying {} over {} {}'.format(patch_path,
  63                                                  url,
  64                                                  refspec))
  65         repository.git.apply(patch_path)
  66     else:
  67         log.info('No patch file provided, skipping patching')
  68
  69     return repository
  70
  71
  72 def create_package_info_file(output_file, repository_list, tag):
  73     """
  74     Generates text file in json format containing basic information about the build
  75     :param output_file:
  76     :param repository_list: list of repositories to be included in package info
  77     :param tag: build version of packages
  78     :return:
  79     """
  80     log.info('Generating package.info file')
  81     build_info = {
  82         'Build_info': {
  83             'build_date': datetime.now().strftime('%Y-%m-%d_%H-%M'),
  84             'Version': tag
  85         }
  86     }
  87     for repository in repository_list:
  88         build_info['Build_info'][
  89             repository.config_reader().get_value('remote "origin"', 'url')] = repository.head.commit.hexsha
  90
  91     with open(output_file, 'w') as outfile:
  92         json.dump(build_info, outfile, indent=4)
  93
  94
  95 def create_package(tar_content, file_name):
  96     """
  97     Creates packages
  98     :param tar_content: list of dictionaries defining src file and destination tar file
  99     :param file_name: output file
 100     """
 101     log.info('Creating package {}'.format(file_name))
 102     with tarfile.open(file_name, 'w') as output_tar_file:
 103         for src, dst in tar_content.items():
 104             if src != '':
 105                 output_tar_file.add(src, dst)
 106
 107
 108 def build_offline_deliverables(build_version,
 109                                application_repository_url,
 110                                application_repository_reference,
 111                                application_patch_file,
 112                                application_charts_dir,
 113                                application_configuration,
 114                                application_patch_role,
 115                                inventory_file,
 116                                output_dir,
 117                                resources_directory,
 118                                aux_directory,
 119                                skip_sw,
 120                                skip_resources,
 121                                skip_aux,
 122                                overwrite):
 123     """
 124     Prepares offline deliverables
 125     :param build_version: Version for packages tagging
 126     :param application_repository_url: git repository hosting application helm charts
 127     :param application_repository_reference: git refspec for repository hosting application helm charts
 128     :param application_patch_file: git patch file to be applied over application repository
 129     :param application_charts_dir: path to directory under application repository containing helm charts
 130     :param application_configuration:  path to application configuration file (helm override configuration)
 131     :param application_patch_role: path to application patch role (executed just before helm deploy)
 132     :param inventory_file: path to ansible inventory file
 133     :param output_dir: Destination directory for saving packages
 134     :param resources_directory: Path to resource directory
 135     :param aux_directory: Path to aux binary directory
 136     :param skip_sw: skip sw package generation
 137     :param skip_resources: skip resources package generation
 138     :param skip_aux: skip aux package generation
 139     :param overwrite: overwrite files in output directory
 140     :return:
 141     """
 142
 143     if os.path.exists(output_dir) and os.listdir(output_dir):
 144         if not overwrite:
 145             log.error('Output directory is not empty, use overwrite to force build')
 146             raise FileExistsError(output_dir)
 147         shutil.rmtree(output_dir)
 148
 149     # Git
 150     offline_repository = git.Repo(offline_repository_dir)
 151
 152     application_dir = os.path.join(output_dir, 'application_repository')
 153     application_repository = prepare_application_repository(application_dir,
 154                                                             application_repository_url,
 155                                                             application_repository_reference,
 156                                                             application_patch_file)
 157
 158     # Package info
 159     info_file = os.path.join(output_dir, 'package.info')
 160     create_package_info_file(info_file, [application_repository, offline_repository], build_version)
 161
 162     # packages layout as dictionaries. <file> : <file location under tar archive>
 163     sw_content = {
 164         os.path.join(offline_repository_dir, 'ansible'): 'ansible',
 165         application_configuration: 'ansible/application/application_configuration.yml',
 166         application_patch_role: 'ansible/application/onap-patch-role',
 167         inventory_file: 'ansible/application/hosts.yml',
 168         os.path.join(application_dir, application_charts_dir): 'ansible/application/helm_charts',
 169         info_file: 'package.info'
 170     }
 171     resources_content = {
 172         resources_directory: '',
 173         info_file: 'package.info'
 174     }
 175     aux_content = {
 176         aux_directory: '',
 177         info_file: 'package.info'
 178     }
 179
 180     # add separator if build version not empty
 181     build_version = "-" + build_version if build_version != "" else ""
 182
 183     if not skip_sw:
 184         log.info('Building offline installer')
 185         os.chdir(os.path.join(offline_repository_dir, 'ansible', 'docker'))
 186         installer_build = subprocess.run(
 187             os.path.join(offline_repository_dir, 'ansible', 'docker', 'build_ansible_image.sh'))
 188         installer_build.check_returncode()
 189         os.chdir(script_location)
 190         sw_package_tar_path = os.path.join(output_dir, 'sw_package' + build_version + '.tar')
 191         create_package(sw_content, sw_package_tar_path)
 192
 193     if not skip_resources:
 194         log.info('Building own dns image')
 195         dns_build = subprocess.run([
 196             os.path.join(offline_repository_dir, 'build', 'creating_data', 'create_nginx_image', '01create-image.sh'),
 197             os.path.join(resources_directory, 'offline_data', 'docker_images_infra')])
 198         dns_build.check_returncode()
 199
 200         # Workaround for downloading without "flat" option
 201         log.info('Binaries - workaround')
 202         download_dir_path = os.path.join(resources_directory, 'downloads')
 203         os.chdir(download_dir_path)
 204         for file in os.listdir(download_dir_path):
 205             if os.path.islink(file):
 206                 os.unlink(file)
 207
 208         rke_files = glob.glob(os.path.join('.', '**/rke_linux-amd64'), recursive=True)
 209         os.symlink(rke_files[0], os.path.join(download_dir_path, rke_files[0].split('/')[-1]))
 210
 211         helm_tar_files = glob.glob(os.path.join('.', '**/helm-*-linux-amd64.tar.gz'), recursive=True)
 212         os.symlink(helm_tar_files[0], os.path.join(download_dir_path, helm_tar_files[0].split('/')[-1]))
 213
 214         kubectl_files = glob.glob(os.path.join('.', '**/kubectl'), recursive=True)
 215         os.symlink(kubectl_files[0], os.path.join(download_dir_path, kubectl_files[0].split('/')[-1]))
 216
 217         os.chdir(script_location)
 218         # End of workaround
 219
 220         resources_package_tar_path = os.path.join(output_dir, 'resources_package' + build_version + '.tar')
 221         create_package(resources_content, resources_package_tar_path)
 222
 223     if not skip_aux:
 224         aux_package_tar_path = os.path.join(output_dir, 'aux_package' + build_version + '.tar')
 225         create_package(aux_content, aux_package_tar_path)
 226
 227     shutil.rmtree(application_dir)
 228
 229
 230 def run_cli():
 231     """
 232     Run as cli tool
 233     """
 234     parser = argparse.ArgumentParser(description='Create Package For Offline Installer')
 235     parser.add_argument('--build-version',
 236                         help='version of the build', default='')
 237     parser.add_argument('application_repository_url', metavar='application-repository-url',
 238                         help='git repository hosting application helm charts')
 239     parser.add_argument('--application-repository_reference', default='master',
 240                         help='git refspec for repository hosting application helm charts')
 241     parser.add_argument('--application-patch_file',
 242                         help='git patch file to be applied over application repository', default='')
 243     parser.add_argument('--application-charts_dir',
 244                         help='path to directory under application repository containing helm charts ',
 245                         default='kubernetes')
 246     parser.add_argument('--application-configuration',
 247                         help='path to application configuration file (helm override configuration)',
 248                         default=os.path.join(offline_repository_dir, 'ansible/config/application_configuration.yml'))
 249     parser.add_argument('--application-patch-role',
 250                         help='path to application patch role file (ansible role) to be executed right before installation',
 251                         default='')
 252     parser.add_argument('--inventory-file', help="path to ansible inventory file",
 253                         default=os.path.join(offline_repository_dir, 'ansible/inventory/hosts.yml'))
 254     parser.add_argument('--output-dir', '-o', default=os.path.join(offline_repository_dir, '../packages'),
 255                         help='Destination directory for saving packages')
 256     parser.add_argument('--resources-directory', default=os.path.join(offline_repository_dir, '../resources'),
 257                         help='Path to resource directory')
 258     parser.add_argument('--aux-directory',
 259                         help='Path to aux binary directory', default='')
 260     parser.add_argument('--skip-sw', action='store_true', default=False,
 261                         help='Set to skip sw package generation')
 262     parser.add_argument('--skip-resources', action='store_true', default=False,
 263                         help='Set to skip resources package generation')
 264     parser.add_argument('--skip-aux', action='store_true', default=False,
 265                         help='Set to skip aux package generation')
 266     parser.add_argument('--overwrite', action='store_true', default=False,
 267                         help='overwrite files in output directory')
 268     parser.add_argument('--debug', action='store_true', default=False,
 269                         help='Turn on debug output')
 270     args = parser.parse_args()
 271
 272     if args.debug:
 273         logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
 274     else:
 275         logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s')
 276
 277     build_offline_deliverables(args.build_version,
 278                                args.application_repository_url,
 279                                args.application_repository_reference,
 280                                args.application_patch_file,
 281                                args.application_charts_dir,
 282                                args.application_configuration,
 283                                args.application_patch_role,
 284                                args.inventory_file,
 285                                args.output_dir,
 286                                args.resources_directory,
 287                                args.aux_directory,
 288                                args.skip_sw,
 289                                args.skip_resources,
 290                                args.skip_aux,
 291                                args.overwrite)
 292
 293
 294 if __name__ == '__main__':
 295     run_cli()