1 #! /usr/bin/env python3
2 # -*- coding: utf-8 -*-
4 # COPYRIGHT NOTICE STARTS HERE
6 # Copyright 2019 . Samsung Electronics Co., Ltd.
8 # Licensed under the Apache License, Version 2.0 (the "License");
9 # you may not use this file except in compliance with the License.
10 # You may obtain a copy of the License at
12 # http://www.apache.org/licenses/LICENSE-2.0
14 # Unless required by applicable law or agreed to in writing, software
15 # distributed under the License is distributed on an "AS IS" BASIS,
16 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 # See the License for the specific language governing permissions and
18 # limitations under the License.
20 # COPYRIGHT NOTICE ENDS HERE
22 from datetime import datetime
36 log = logging.getLogger(__name__)
37 script_location = os.path.abspath(os.path.join(__file__, '..'))
38 offline_repository_dir = os.path.abspath(os.path.join(script_location, '..'))
41 def prepare_application_repository(directory, url, refspec, patch_path):
43 Downloads git repository according to refspec, applies patch if provided
44 :param directory: path to repository
45 :param url: url to repository
46 :param refspec: refspec to fetch
47 :param patch_path: path git patch to be applied over repository
48 :return: repository - git repository object
52 shutil.rmtree(directory)
53 except FileNotFoundError:
56 log.info('Cloning {} with refspec {} '.format(url, refspec))
57 repository = git.Repo.init(directory)
58 origin = repository.create_remote('origin', url)
60 repository.git.submodule('update', '--init')
63 log.info('Applying {} over {} {}'.format(patch_path,
66 repository.git.apply(patch_path)
68 log.info('No patch file provided, skipping patching')
73 def create_package_info_file(output_file, repository_list, tag, metadata):
75 Generates text file in json format containing basic information about the build
77 :param repository_list: list of repositories to be included in package info
78 :param tag: build version of packages
79 :param metadata: additional metadata into package.info
82 log.info('Generating package.info file')
85 'build_date': datetime.now().strftime('%Y-%m-%d_%H-%M'),
90 for repository in repository_list:
91 build_info['Build_info'][
92 repository.config_reader().get_value('remote "origin"', 'url')] = repository.head.commit.hexsha
96 build_info['Build_info'].update(meta)
98 with open(output_file, 'w') as outfile:
99 json.dump(build_info, outfile, indent=4)
102 def add_checksum_info(output_dir):
104 Add checksum information into package.info file
105 :param output_dir: directory where are packages
107 tar_files = ['resources_package.tar', 'aux_package.tar', 'sw_package.tar']
108 for tar_file in tar_files:
110 checksum = hashlib.md5()
111 with open(os.path.join(output_dir, tar_file), 'rb') as f:
112 for chunk in iter(lambda: f.read(4096), b""):
113 checksum.update(chunk)
114 with open(os.path.join(output_dir, 'package.info'), 'r') as f:
115 json_data = json.load(f)
116 json_data['Build_info']['Packages'].update({tar_file: checksum.hexdigest()})
117 with open(os.path.join(output_dir, 'package.info'), 'w') as f:
118 json.dump(json_data, f, indent=4)
119 except FileNotFoundError:
123 def create_package(tar_content, file_name):
126 :param tar_content: list of dictionaries defining src file and destination tar file
127 :param file_name: output file
129 log.info('Creating package {}'.format(file_name))
130 with tarfile.open(file_name, 'w') as output_tar_file:
131 for src, dst in tar_content.items():
133 output_tar_file.add(src, dst)
136 def metadata_validation(param):
138 Validation of metadata parameters
139 :param param: parameter to be checked needs to be in format key=value
142 key, value = param.split('=')
143 assert (key and value)
145 except (ValueError, AssertionError):
146 msg = "%r is not a valid parameter. Needs to be in format key=value" % param
147 raise argparse.ArgumentTypeError(msg)
150 def build_offline_deliverables(build_version,
151 application_repository_url,
152 application_repository_reference,
153 application_patch_file,
154 application_charts_dir,
155 application_configuration,
156 application_patch_role,
166 Prepares offline deliverables
167 :param build_version: Version for packages tagging
168 :param application_repository_url: git repository hosting application helm charts
169 :param application_repository_reference: git refspec for repository hosting application helm charts
170 :param application_patch_file: git patch file to be applied over application repository
171 :param application_charts_dir: path to directory under application repository containing helm charts
172 :param application_configuration: path to application configuration file (helm override configuration)
173 :param application_patch_role: path to application patch role (executed just before helm deploy)
174 :param output_dir: Destination directory for saving packages
175 :param resources_directory: Path to resource directory
176 :param aux_directory: Path to aux binary directory
177 :param skip_sw: skip sw package generation
178 :param skip_resources: skip resources package generation
179 :param skip_aux: skip aux package generation
180 :param overwrite: overwrite files in output directory
181 :param metadata: add metadata info into package.info
185 if os.path.exists(output_dir) and os.listdir(output_dir):
187 log.error('Output directory is not empty, use overwrite to force build')
188 raise FileExistsError(output_dir)
189 shutil.rmtree(output_dir)
192 offline_repository = git.Repo(offline_repository_dir)
194 application_dir = os.path.join(output_dir, 'application_repository')
195 application_repository = prepare_application_repository(application_dir,
196 application_repository_url,
197 application_repository_reference,
198 application_patch_file)
201 info_file = os.path.join(output_dir, 'package.info')
202 create_package_info_file(info_file, [application_repository, offline_repository], build_version, metadata)
204 # packages layout as dictionaries. <file> : <file location under tar archive>
206 os.path.join(offline_repository_dir, 'ansible'): 'ansible',
207 application_configuration: 'ansible/application/application_configuration.yml',
208 application_patch_role: 'ansible/application/onap-patch-role',
209 os.path.join(application_dir, application_charts_dir): 'ansible/application/helm_charts',
210 info_file: 'package.info'
212 resources_content = {
213 resources_directory: '',
214 info_file: 'package.info'
218 info_file: 'package.info'
222 log.info('Building offline installer')
223 os.chdir(os.path.join(offline_repository_dir, 'ansible', 'docker'))
224 installer_build = subprocess.run(
225 os.path.join(offline_repository_dir, 'ansible', 'docker', 'build_ansible_image.sh'))
226 installer_build.check_returncode()
227 os.chdir(script_location)
228 sw_package_tar_path = os.path.join(output_dir, 'sw_package.tar')
229 create_package(sw_content, sw_package_tar_path)
231 if not skip_resources:
232 log.info('Building own dns image')
233 dns_build = subprocess.run([
234 os.path.join(offline_repository_dir, 'build', 'creating_data', 'create_nginx_image', '01create-image.sh'),
235 os.path.join(resources_directory, 'offline_data', 'docker_images_infra')])
236 dns_build.check_returncode()
238 # Workaround for downloading without "flat" option
239 log.info('Binaries - workaround')
240 download_dir_path = os.path.join(resources_directory, 'downloads')
241 os.chdir(download_dir_path)
242 for file in os.listdir(download_dir_path):
243 if os.path.islink(file):
246 bin_pattern_list = ['**/rke_linux-amd64',
247 '**/helm-*-linux-amd64.tar.gz',
249 '**/amd64/chartmuseum',
250 '**/helm-push_*_linux_amd64.tar.gz']
252 for pattern in bin_pattern_list:
253 for bin_file in glob.glob(os.path.join('.', pattern), recursive=True):
254 os.symlink(bin_file, os.path.join(download_dir_path, bin_file.split('/')[-1]))
256 os.chdir(script_location)
259 resources_package_tar_path = os.path.join(output_dir, 'resources_package.tar')
260 create_package(resources_content, resources_package_tar_path)
263 aux_package_tar_path = os.path.join(output_dir, 'aux_package.tar')
264 create_package(aux_content, aux_package_tar_path)
266 add_checksum_info(output_dir)
267 shutil.rmtree(application_dir, ignore_errors=True)
274 parser = argparse.ArgumentParser(description='Create Package For Offline Installer')
275 parser.add_argument('--build-version',
276 help='version of the build', default='')
277 parser.add_argument('application_repository_url', metavar='application-repository-url',
278 help='git repository hosting application helm charts')
279 parser.add_argument('--application-repository_reference', default='master',
280 help='git refspec for repository hosting application helm charts')
281 parser.add_argument('--application-patch_file',
282 help='git patch file to be applied over application repository', default='')
283 parser.add_argument('--application-charts_dir',
284 help='path to directory under application repository containing helm charts ',
285 default='kubernetes')
286 parser.add_argument('--application-configuration',
287 help='path to application configuration file (helm override configuration)',
288 default=os.path.join(offline_repository_dir, 'config/application_configuration.yml'))
289 parser.add_argument('--application-patch-role',
290 help='path to application patch role file (ansible role) to be executed right before installation',
292 parser.add_argument('--output-dir', '-o', default=os.path.join(offline_repository_dir, '../packages'),
293 help='Destination directory for saving packages')
294 parser.add_argument('--resources-directory', default=os.path.join(offline_repository_dir, '../resources'),
295 help='Path to resource directory')
296 parser.add_argument('--aux-directory',
297 help='Path to aux binary directory', default='')
298 parser.add_argument('--skip-sw', action='store_true', default=False,
299 help='Set to skip sw package generation')
300 parser.add_argument('--skip-resources', action='store_true', default=False,
301 help='Set to skip resources package generation')
302 parser.add_argument('--skip-aux', action='store_true', default=False,
303 help='Set to skip aux package generation')
304 parser.add_argument('--overwrite', action='store_true', default=False,
305 help='overwrite files in output directory')
306 parser.add_argument('--debug', action='store_true', default=False,
307 help='Turn on debug output')
308 parser.add_argument('--add-metadata', nargs="+", type=metadata_validation,
309 help='additional metadata added into package.info, format: key=value')
310 args = parser.parse_args()
313 logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
315 logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s')
317 build_offline_deliverables(args.build_version,
318 args.application_repository_url,
319 args.application_repository_reference,
320 args.application_patch_file,
321 args.application_charts_dir,
322 args.application_configuration,
323 args.application_patch_role,
325 args.resources_directory,
334 if __name__ == '__main__':