build/package.py

   1 #! /usr/bin/env python3
   2 # -*- coding: utf-8 -*-
   3
   4 #   COPYRIGHT NOTICE STARTS HERE
   5
   6 #   Copyright 2019 . Samsung Electronics Co., Ltd.
   7 #
   8 #   Licensed under the Apache License, Version 2.0 (the "License");
   9 #   you may not use this file except in compliance with the License.
  10 #   You may obtain a copy of the License at
  11 #
  12 #       http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 #   Unless required by applicable law or agreed to in writing, software
  15 #   distributed under the License is distributed on an "AS IS" BASIS,
  16 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17 #   See the License for the specific language governing permissions and
  18 #   limitations under the License.
  19
  20 #   COPYRIGHT NOTICE ENDS HERE
  21
  22 from datetime import datetime
  23 import subprocess
  24 import argparse
  25 import logging
  26 import shutil
  27 import glob
  28 import json
  29 import sys
  30 import os
  31
  32 import tarfile
  33 import git
  34
  35 log = logging.getLogger(__name__)
  36 script_location = os.path.abspath(os.path.join(__file__, '..'))
  37 offline_repository_dir = os.path.abspath(os.path.join(script_location, '..'))
  38
  39
  40 def prepare_application_repository(directory, url, refspec, patch_path):
  41     """
  42     Downloads git repository according to refspec, applies patch if provided
  43     :param directory: path to repository
  44     :param url: url to repository
  45     :param refspec: refspec to fetch
  46     :param patch_path: path git patch to be applied over repository
  47     :return: repository - git repository object
  48     """
  49
  50     try:
  51         shutil.rmtree(directory)
  52     except FileNotFoundError:
  53         pass
  54
  55     log.info('Cloning {} with refspec {} '.format(url, refspec))
  56     repository = git.Repo.init(directory)
  57     origin = repository.create_remote('origin', url)
  58     origin.pull(refspec)
  59     repository.git.submodule('update', '--init')
  60
  61     if patch_path:
  62         log.info('Applying {} over {} {}'.format(patch_path,
  63                                                  url,
  64                                                  refspec))
  65         repository.git.apply(patch_path)
  66     else:
  67         log.info('No patch file provided, skipping patching')
  68
  69     return repository
  70
  71
  72 def create_package_info_file(output_file, repository_list, tag):
  73     """
  74     Generates text file in json format containing basic information about the build
  75     :param output_file:
  76     :param repository_list: list of repositories to be included in package info
  77     :param tag: build version of packages
  78     :return:
  79     """
  80     log.info('Generating package.info file')
  81     build_info = {
  82         'Build_info': {
  83             'build_date': datetime.now().strftime('%Y-%m-%d_%H-%M'),
  84             'Version': tag
  85         }
  86     }
  87     for repository in repository_list:
  88         build_info['Build_info'][
  89             repository.config_reader().get_value('remote "origin"', 'url')] = repository.head.commit.hexsha
  90
  91     with open(output_file, 'w') as outfile:
  92         json.dump(build_info, outfile, indent=4)
  93
  94
  95 def create_package(tar_content, file_name):
  96     """
  97     Creates packages
  98     :param tar_content: list of dictionaries defining src file and destination tar file
  99     :param file_name: output file
 100     """
 101     log.info('Creating package {}'.format(file_name))
 102     with tarfile.open(file_name, 'w') as output_tar_file:
 103         for src, dst in tar_content.items():
 104             if src != '':
 105                 output_tar_file.add(src, dst)
 106
 107
 108 def build_offline_deliverables(build_version,
 109                                application_repository_url,
 110                                application_repository_reference,
 111                                application_patch_file,
 112                                application_charts_dir,
 113                                application_configuration,
 114                                application_patch_role,
 115                                output_dir,
 116                                resources_directory,
 117                                aux_directory,
 118                                skip_sw,
 119                                skip_resources,
 120                                skip_aux,
 121                                overwrite):
 122     """
 123     Prepares offline deliverables
 124     :param build_version: Version for packages tagging
 125     :param application_repository_url: git repository hosting application helm charts
 126     :param application_repository_reference: git refspec for repository hosting application helm charts
 127     :param application_patch_file: git patch file to be applied over application repository
 128     :param application_charts_dir: path to directory under application repository containing helm charts
 129     :param application_configuration:  path to application configuration file (helm override configuration)
 130     :param application_patch_role: path to application patch role (executed just before helm deploy)
 131     :param output_dir: Destination directory for saving packages
 132     :param resources_directory: Path to resource directory
 133     :param aux_directory: Path to aux binary directory
 134     :param skip_sw: skip sw package generation
 135     :param skip_resources: skip resources package generation
 136     :param skip_aux: skip aux package generation
 137     :param overwrite: overwrite files in output directory
 138     :return:
 139     """
 140
 141     if os.path.exists(output_dir) and os.listdir(output_dir):
 142         if not overwrite:
 143             log.error('Output directory is not empty, use overwrite to force build')
 144             raise FileExistsError(output_dir)
 145         shutil.rmtree(output_dir)
 146
 147     # Git
 148     offline_repository = git.Repo(offline_repository_dir)
 149
 150     application_dir = os.path.join(output_dir, 'application_repository')
 151     application_repository = prepare_application_repository(application_dir,
 152                                                             application_repository_url,
 153                                                             application_repository_reference,
 154                                                             application_patch_file)
 155
 156     # Package info
 157     info_file = os.path.join(output_dir, 'package.info')
 158     create_package_info_file(info_file, [application_repository, offline_repository], build_version)
 159
 160     # packages layout as dictionaries. <file> : <file location under tar archive>
 161     sw_content = {
 162         os.path.join(offline_repository_dir, 'ansible'): 'ansible',
 163         application_configuration: 'ansible/application/application_configuration.yml',
 164         application_patch_role: 'ansible/application/onap-patch-role',
 165         os.path.join(application_dir, application_charts_dir): 'ansible/application/helm_charts',
 166         info_file: 'package.info'
 167     }
 168     resources_content = {
 169         resources_directory: '',
 170         info_file: 'package.info'
 171     }
 172     aux_content = {
 173         aux_directory: '',
 174         info_file: 'package.info'
 175     }
 176
 177     # add separator if build version not empty
 178     build_version = "-" + build_version if build_version != "" else ""
 179
 180     if not skip_sw:
 181         log.info('Building offline installer')
 182         os.chdir(os.path.join(offline_repository_dir, 'ansible', 'docker'))
 183         installer_build = subprocess.run(
 184             os.path.join(offline_repository_dir, 'ansible', 'docker', 'build_ansible_image.sh'))
 185         installer_build.check_returncode()
 186         os.chdir(script_location)
 187         sw_package_tar_path = os.path.join(output_dir, 'sw_package' + build_version + '.tar')
 188         create_package(sw_content, sw_package_tar_path)
 189
 190     if not skip_resources:
 191         log.info('Building own dns image')
 192         dns_build = subprocess.run([
 193             os.path.join(offline_repository_dir, 'build', 'creating_data', 'create_nginx_image', '01create-image.sh'),
 194             os.path.join(resources_directory, 'offline_data', 'docker_images_infra')])
 195         dns_build.check_returncode()
 196
 197         # Workaround for downloading without "flat" option
 198         log.info('Binaries - workaround')
 199         download_dir_path = os.path.join(resources_directory, 'downloads')
 200         os.chdir(download_dir_path)
 201         for file in os.listdir(download_dir_path):
 202             if os.path.islink(file):
 203                 os.unlink(file)
 204
 205         rke_files = glob.glob(os.path.join('.', '**/rke_linux-amd64'), recursive=True)
 206         os.symlink(rke_files[0], os.path.join(download_dir_path, rke_files[0].split('/')[-1]))
 207
 208         helm_tar_files = glob.glob(os.path.join('.', '**/helm-*-linux-amd64.tar.gz'), recursive=True)
 209         os.symlink(helm_tar_files[0], os.path.join(download_dir_path, helm_tar_files[0].split('/')[-1]))
 210
 211         kubectl_files = glob.glob(os.path.join('.', '**/kubectl'), recursive=True)
 212         os.symlink(kubectl_files[0], os.path.join(download_dir_path, kubectl_files[0].split('/')[-1]))
 213
 214         os.chdir(script_location)
 215         # End of workaround
 216
 217         resources_package_tar_path = os.path.join(output_dir, 'resources_package' + build_version + '.tar')
 218         create_package(resources_content, resources_package_tar_path)
 219
 220     if not skip_aux:
 221         aux_package_tar_path = os.path.join(output_dir, 'aux_package' + build_version + '.tar')
 222         create_package(aux_content, aux_package_tar_path)
 223
 224     shutil.rmtree(application_dir)
 225
 226
 227 def run_cli():
 228     """
 229     Run as cli tool
 230     """
 231     parser = argparse.ArgumentParser(description='Create Package For Offline Installer')
 232     parser.add_argument('--build-version',
 233                         help='version of the build', default='')
 234     parser.add_argument('application_repository_url', metavar='application-repository-url',
 235                         help='git repository hosting application helm charts')
 236     parser.add_argument('--application-repository_reference', default='master',
 237                         help='git refspec for repository hosting application helm charts')
 238     parser.add_argument('--application-patch_file',
 239                         help='git patch file to be applied over application repository', default='')
 240     parser.add_argument('--application-charts_dir',
 241                         help='path to directory under application repository containing helm charts ',
 242                         default='kubernetes')
 243     parser.add_argument('--application-configuration',
 244                         help='path to application configuration file (helm override configuration)',
 245                         default=os.path.join(offline_repository_dir, 'config/application_configuration.yml'))
 246     parser.add_argument('--application-patch-role',
 247                         help='path to application patch role file (ansible role) to be executed right before installation',
 248                         default='')
 249     parser.add_argument('--output-dir', '-o', default=os.path.join(offline_repository_dir, '../packages'),
 250                         help='Destination directory for saving packages')
 251     parser.add_argument('--resources-directory', default=os.path.join(offline_repository_dir, '../resources'),
 252                         help='Path to resource directory')
 253     parser.add_argument('--aux-directory',
 254                         help='Path to aux binary directory', default='')
 255     parser.add_argument('--skip-sw', action='store_true', default=False,
 256                         help='Set to skip sw package generation')
 257     parser.add_argument('--skip-resources', action='store_true', default=False,
 258                         help='Set to skip resources package generation')
 259     parser.add_argument('--skip-aux', action='store_true', default=False,
 260                         help='Set to skip aux package generation')
 261     parser.add_argument('--overwrite', action='store_true', default=False,
 262                         help='overwrite files in output directory')
 263     parser.add_argument('--debug', action='store_true', default=False,
 264                         help='Turn on debug output')
 265     args = parser.parse_args()
 266
 267     if args.debug:
 268         logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
 269     else:
 270         logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s')
 271
 272     build_offline_deliverables(args.build_version,
 273                                args.application_repository_url,
 274                                args.application_repository_reference,
 275                                args.application_patch_file,
 276                                args.application_charts_dir,
 277                                args.application_configuration,
 278                                args.application_patch_role,
 279                                args.output_dir,
 280                                args.resources_directory,
 281                                args.aux_directory,
 282                                args.skip_sw,
 283                                args.skip_resources,
 284                                args.skip_aux,
 285                                args.overwrite)
 286
 287
 288 if __name__ == '__main__':
 289     run_cli()