build/package.py

   1 #! /usr/bin/env python3
   2 # -*- coding: utf-8 -*-
   3
   4 #   COPYRIGHT NOTICE STARTS HERE
   5
   6 #   Copyright 2019 . Samsung Electronics Co., Ltd.
   7 #
   8 #   Licensed under the Apache License, Version 2.0 (the "License");
   9 #   you may not use this file except in compliance with the License.
  10 #   You may obtain a copy of the License at
  11 #
  12 #       http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 #   Unless required by applicable law or agreed to in writing, software
  15 #   distributed under the License is distributed on an "AS IS" BASIS,
  16 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17 #   See the License for the specific language governing permissions and
  18 #   limitations under the License.
  19
  20 #   COPYRIGHT NOTICE ENDS HERE
  21
  22 from datetime import datetime
  23 import subprocess
  24 import argparse
  25 import logging
  26 import shutil
  27 import glob
  28 import json
  29 import sys
  30 import os
  31 import hashlib
  32
  33 import tarfile
  34 import git
  35
  36 log = logging.getLogger(__name__)
  37 script_location = os.path.abspath(os.path.join(__file__, '..'))
  38 offline_repository_dir = os.path.abspath(os.path.join(script_location, '..'))
  39
  40
  41 def prepare_application_repository(directory, url, refspec, patch_path):
  42     """
  43     Downloads git repository according to refspec, applies patch if provided
  44     :param directory: path to repository
  45     :param url: url to repository
  46     :param refspec: refspec to fetch
  47     :param patch_path: path git patch to be applied over repository
  48     :return: repository - git repository object
  49     """
  50
  51     try:
  52         shutil.rmtree(directory)
  53     except FileNotFoundError:
  54         pass
  55
  56     log.info('Cloning {} with refspec {} '.format(url, refspec))
  57     repository = git.Repo.init(directory)
  58     origin = repository.create_remote('origin', url)
  59     origin.pull(refspec)
  60     repository.git.submodule('update', '--init')
  61
  62     if patch_path:
  63         log.info('Applying {} over {} {}'.format(patch_path,
  64                                                  url,
  65                                                  refspec))
  66         repository.git.apply(patch_path)
  67     else:
  68         log.info('No patch file provided, skipping patching')
  69
  70     return repository
  71
  72
  73 def create_package_info_file(output_file, repository_list, tag, metadata):
  74     """
  75     Generates text file in json format containing basic information about the build
  76     :param output_file:
  77     :param repository_list: list of repositories to be included in package info
  78     :param tag: build version of packages
  79     :param metadata: additional metadata into package.info
  80     :return:
  81     """
  82     log.info('Generating package.info file')
  83     build_info = {
  84         'Build_info': {
  85             'build_date': datetime.now().strftime('%Y-%m-%d_%H-%M'),
  86             'Version': tag,
  87             'Packages': {}
  88         }
  89     }
  90     for repository in repository_list:
  91         build_info['Build_info'][
  92             repository.config_reader().get_value('remote "origin"', 'url')] = repository.head.commit.hexsha
  93
  94     if metadata:
  95         for meta in metadata:
  96             build_info['Build_info'].update(meta)
  97
  98     with open(output_file, 'w') as outfile:
  99         json.dump(build_info, outfile, indent=4)
 100
 101
 102 def add_checksum_info(output_dir):
 103     """
 104     Add checksum information into package.info file
 105     :param output_dir: directory where are packages
 106     """
 107     tar_files = ['resources_package.tar', 'aux_package.tar', 'sw_package.tar']
 108     for tar_file in tar_files:
 109         try:
 110             data = os.path.join(output_dir, tar_file)
 111             cksum = hashlib.md5(open(data, 'rb').read()).hexdigest()
 112             with open(os.path.join(output_dir, 'package.info'), 'r') as f:
 113                 json_data = json.load(f)
 114                 json_data['Build_info']['Packages'].update({tar_file: cksum})
 115             with open(os.path.join(output_dir, 'package.info'), 'w') as f:
 116                 json.dump(json_data, f, indent=4)
 117         except FileNotFoundError:
 118             pass
 119
 120
 121 def create_package(tar_content, file_name):
 122     """
 123     Creates packages
 124     :param tar_content: list of dictionaries defining src file and destination tar file
 125     :param file_name: output file
 126     """
 127     log.info('Creating package {}'.format(file_name))
 128     with tarfile.open(file_name, 'w') as output_tar_file:
 129         for src, dst in tar_content.items():
 130             if src != '':
 131                 output_tar_file.add(src, dst)
 132
 133
 134 def metadata_validation(param):
 135     """
 136     Validation of metadata parameters
 137     :param param: parameter to be checked needs to be in format key=value
 138     """
 139     try:
 140         key, value = param.split('=')
 141         assert (key and value)
 142         return {key: value}
 143     except (ValueError, AssertionError):
 144         msg = "%r is not a valid parameter. Needs to be in format key=value" % param
 145         raise argparse.ArgumentTypeError(msg)
 146
 147
 148 def build_offline_deliverables(build_version,
 149                                application_repository_url,
 150                                application_repository_reference,
 151                                application_patch_file,
 152                                application_charts_dir,
 153                                application_configuration,
 154                                application_patch_role,
 155                                output_dir,
 156                                resources_directory,
 157                                aux_directory,
 158                                skip_sw,
 159                                skip_resources,
 160                                skip_aux,
 161                                overwrite,
 162                                metadata):
 163     """
 164     Prepares offline deliverables
 165     :param build_version: Version for packages tagging
 166     :param application_repository_url: git repository hosting application helm charts
 167     :param application_repository_reference: git refspec for repository hosting application helm charts
 168     :param application_patch_file: git patch file to be applied over application repository
 169     :param application_charts_dir: path to directory under application repository containing helm charts
 170     :param application_configuration:  path to application configuration file (helm override configuration)
 171     :param application_patch_role: path to application patch role (executed just before helm deploy)
 172     :param output_dir: Destination directory for saving packages
 173     :param resources_directory: Path to resource directory
 174     :param aux_directory: Path to aux binary directory
 175     :param skip_sw: skip sw package generation
 176     :param skip_resources: skip resources package generation
 177     :param skip_aux: skip aux package generation
 178     :param overwrite: overwrite files in output directory
 179     :param metadata: add metadata info into package.info
 180     :return:
 181     """
 182
 183     if os.path.exists(output_dir) and os.listdir(output_dir):
 184         if not overwrite:
 185             log.error('Output directory is not empty, use overwrite to force build')
 186             raise FileExistsError(output_dir)
 187         shutil.rmtree(output_dir)
 188
 189     # Git
 190     offline_repository = git.Repo(offline_repository_dir)
 191
 192     application_dir = os.path.join(output_dir, 'application_repository')
 193     application_repository = prepare_application_repository(application_dir,
 194                                                             application_repository_url,
 195                                                             application_repository_reference,
 196                                                             application_patch_file)
 197
 198     # Package info
 199     info_file = os.path.join(output_dir, 'package.info')
 200     create_package_info_file(info_file, [application_repository, offline_repository], build_version, metadata)
 201
 202     # packages layout as dictionaries. <file> : <file location under tar archive>
 203     sw_content = {
 204         os.path.join(offline_repository_dir, 'ansible'): 'ansible',
 205         application_configuration: 'ansible/application/application_configuration.yml',
 206         application_patch_role: 'ansible/application/onap-patch-role',
 207         os.path.join(application_dir, application_charts_dir): 'ansible/application/helm_charts',
 208         info_file: 'package.info'
 209     }
 210     resources_content = {
 211         resources_directory: '',
 212         info_file: 'package.info'
 213     }
 214     aux_content = {
 215         aux_directory: '',
 216         info_file: 'package.info'
 217     }
 218
 219     if not skip_sw:
 220         log.info('Building offline installer')
 221         os.chdir(os.path.join(offline_repository_dir, 'ansible', 'docker'))
 222         installer_build = subprocess.run(
 223             os.path.join(offline_repository_dir, 'ansible', 'docker', 'build_ansible_image.sh'))
 224         installer_build.check_returncode()
 225         os.chdir(script_location)
 226         sw_package_tar_path = os.path.join(output_dir, 'sw_package.tar')
 227         create_package(sw_content, sw_package_tar_path)
 228
 229     if not skip_resources:
 230         log.info('Building own dns image')
 231         dns_build = subprocess.run([
 232             os.path.join(offline_repository_dir, 'build', 'creating_data', 'create_nginx_image', '01create-image.sh'),
 233             os.path.join(resources_directory, 'offline_data', 'docker_images_infra')])
 234         dns_build.check_returncode()
 235
 236         # Workaround for downloading without "flat" option
 237         log.info('Binaries - workaround')
 238         download_dir_path = os.path.join(resources_directory, 'downloads')
 239         os.chdir(download_dir_path)
 240         for file in os.listdir(download_dir_path):
 241             if os.path.islink(file):
 242                 os.unlink(file)
 243
 244         rke_files = glob.glob(os.path.join('.', '**/rke_linux-amd64'), recursive=True)
 245         os.symlink(rke_files[0], os.path.join(download_dir_path, rke_files[0].split('/')[-1]))
 246
 247         helm_tar_files = glob.glob(os.path.join('.', '**/helm-*-linux-amd64.tar.gz'), recursive=True)
 248         os.symlink(helm_tar_files[0], os.path.join(download_dir_path, helm_tar_files[0].split('/')[-1]))
 249
 250         kubectl_files = glob.glob(os.path.join('.', '**/kubectl'), recursive=True)
 251         os.symlink(kubectl_files[0], os.path.join(download_dir_path, kubectl_files[0].split('/')[-1]))
 252
 253         os.chdir(script_location)
 254         # End of workaround
 255
 256         resources_package_tar_path = os.path.join(output_dir, 'resources_package.tar')
 257         create_package(resources_content, resources_package_tar_path)
 258
 259     if not skip_aux:
 260         aux_package_tar_path = os.path.join(output_dir, 'aux_package.tar')
 261         create_package(aux_content, aux_package_tar_path)
 262
 263     add_checksum_info(output_dir)
 264     shutil.rmtree(application_dir)
 265
 266
 267 def run_cli():
 268     """
 269     Run as cli tool
 270     """
 271     parser = argparse.ArgumentParser(description='Create Package For Offline Installer')
 272     parser.add_argument('--build-version',
 273                         help='version of the build', default='')
 274     parser.add_argument('application_repository_url', metavar='application-repository-url',
 275                         help='git repository hosting application helm charts')
 276     parser.add_argument('--application-repository_reference', default='master',
 277                         help='git refspec for repository hosting application helm charts')
 278     parser.add_argument('--application-patch_file',
 279                         help='git patch file to be applied over application repository', default='')
 280     parser.add_argument('--application-charts_dir',
 281                         help='path to directory under application repository containing helm charts ',
 282                         default='kubernetes')
 283     parser.add_argument('--application-configuration',
 284                         help='path to application configuration file (helm override configuration)',
 285                         default=os.path.join(offline_repository_dir, 'config/application_configuration.yml'))
 286     parser.add_argument('--application-patch-role',
 287                         help='path to application patch role file (ansible role) to be executed right before installation',
 288                         default='')
 289     parser.add_argument('--output-dir', '-o', default=os.path.join(offline_repository_dir, '../packages'),
 290                         help='Destination directory for saving packages')
 291     parser.add_argument('--resources-directory', default=os.path.join(offline_repository_dir, '../resources'),
 292                         help='Path to resource directory')
 293     parser.add_argument('--aux-directory',
 294                         help='Path to aux binary directory', default='')
 295     parser.add_argument('--skip-sw', action='store_true', default=False,
 296                         help='Set to skip sw package generation')
 297     parser.add_argument('--skip-resources', action='store_true', default=False,
 298                         help='Set to skip resources package generation')
 299     parser.add_argument('--skip-aux', action='store_true', default=False,
 300                         help='Set to skip aux package generation')
 301     parser.add_argument('--overwrite', action='store_true', default=False,
 302                         help='overwrite files in output directory')
 303     parser.add_argument('--debug', action='store_true', default=False,
 304                         help='Turn on debug output')
 305     parser.add_argument('--add-metadata', nargs="+", type=metadata_validation,
 306                         help='additional metadata added into package.info, format: key=value')
 307     args = parser.parse_args()
 308
 309     if args.debug:
 310         logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
 311     else:
 312         logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s')
 313
 314     build_offline_deliverables(args.build_version,
 315                                args.application_repository_url,
 316                                args.application_repository_reference,
 317                                args.application_patch_file,
 318                                args.application_charts_dir,
 319                                args.application_configuration,
 320                                args.application_patch_role,
 321                                args.output_dir,
 322                                args.resources_directory,
 323                                args.aux_directory,
 324                                args.skip_sw,
 325                                args.skip_resources,
 326                                args.skip_aux,
 327                                args.overwrite,
 328                                args.add_metadata)
 329
 330
 331 if __name__ == '__main__':
 332     run_cli()