From 0d19d3bfb1d47d3dcd5fe5a6db3a8445c5a1007c Mon Sep 17 00:00:00 2001 From: efiacor Date: Thu, 1 Dec 2022 14:52:39 +0000 Subject: [PATCH] [HELM] Clean up and fix oom helm jobs Change-Id: I44c198e86f09f06d4ccd6e8b89c111ee49ebee2c Signed-off-by: efiacor Issue-ID: OOM-3065 --- jjb/global-templates-helm.yaml | 101 +++++++++++++++++--------------- jjb/oom/oom-helm.yaml | 59 +++++++++++-------- jjb/oom/oom-main-linter.yaml | 1 + shell/apply-submodule-patch-oom.sh | 18 ------ shell/helm/helm-install.sh | 14 +++++ shell/{ => helm}/helm-repo-init.sh | 1 - shell/helm/make-helm-all.sh | 5 ++ shell/helm/make-helm-skip-lint.sh | 5 ++ shell/{ => helm}/oom-create-netrc.sh | 0 shell/{ => helm}/publish_helm_charts.sh | 10 +--- 10 files changed, 117 insertions(+), 97 deletions(-) delete mode 100644 shell/apply-submodule-patch-oom.sh create mode 100755 shell/helm/helm-install.sh rename shell/{ => helm}/helm-repo-init.sh (95%) create mode 100755 shell/helm/make-helm-all.sh create mode 100755 shell/helm/make-helm-skip-lint.sh rename shell/{ => helm}/oom-create-netrc.sh (100%) rename shell/{ => helm}/publish_helm_charts.sh (67%) diff --git a/jjb/global-templates-helm.yaml b/jjb/global-templates-helm.yaml index 3ce9285ec..7b74b0eac 100644 --- a/jjb/global-templates-helm.yaml +++ b/jjb/global-templates-helm.yaml @@ -8,7 +8,14 @@ builders: - inject: properties-content: "SERVER_ID={server-id}" - - shell: !include-raw-escape: ../shell/oom-create-netrc.sh + - shell: !include-raw-escape: ../shell/helm/oom-create-netrc.sh + +- builder: + name: helm-install + builders: + - inject: + properties-content: "HELM_VER={helm_ver}" + - shell: !include-raw-escape: ../shell/helm/helm-install.sh ########### # ANCHORS # @@ -19,17 +26,12 @@ ###################### # Default parameters # ###################### - branch: master - submodule-recursive: true - # default params for helm jobs build_script: '' git-url: '$GIT_URL/$PROJECT' post_build_script: '' pre_build_script: '' - submodule-disable: false - submodule-timeout: 10 build-timeout: 240 ##################### @@ -49,12 +51,30 @@ project: '{project}' stream: '{stream}' lftools-version: '{lftools-version}' + - string: + name: HELM_BIN + default: '{helm_bin|helm3.8}' + - string: + name: BUILD_TYPE + default: '{build_type|snapshot}' wrappers: - lf-infra-wrappers: build-timeout: '{build-timeout}' jenkins-ssh-credential: '{jenkins-ssh-credential}' + builders: + - lf-infra-pre-build + - lf-provide-maven-settings: + global-settings-file: '{mvn-global-settings}' + settings-file: '{mvn-settings}' + - shell: '{obj:pre_build_script}' + - shell: '{obj:build_script}' + - oom-create-netrc: + server-id: oom-helm + - shell: '{obj:post_build_script}' + - lf-provide-maven-settings-cleanup + publishers: - lf-infra-publish @@ -76,20 +96,11 @@ triggers: - gerrit-trigger-helm-patch-submitted: - status-context: '{project}-Helm-Verify' server: '{server-name}' project: '{project}' branch: '{branch}' - helm_bin: '{helm_bin}' files: '{files}' - builders: - - lf-infra-pre-build - - shell: | - echo "export HELM_BIN={helm_bin}" > helm.prop - - shell: '{obj:pre_build_script}' - - shell: '{obj:build_script}' - - job-template: name: '{project-name}-{stream}-merge-helm' # Job template for Helm merge jobs @@ -107,26 +118,8 @@ server: '{server-name}' project: '{project}' branch: '{branch}' - helm_bin: '{helm_bin}' files: '{files}' - builders: - - lf-infra-pre-build - - lf-provide-maven-settings: - global-settings-file: '{mvn-global-settings}' - settings-file: '{mvn-settings}' - - shell: | - echo "export HELM_BIN={helm_bin}" > helm.prop - - shell: '{obj:pre_build_script}' - - shell: '{obj:build_script}' - - oom-create-netrc: - server-id: oom-helm - - inject: - properties-content: | - BUILD_TYPE=snapshot - - shell: '{obj:post_build_script}' - - lf-provide-maven-settings-cleanup - - job-template: name: '{project-name}-{stream}-release-helm' # Job template for Helm stage jobs @@ -141,7 +134,6 @@ triggers: - gerrit: - helm_bin: '{helm_bin}' server-name: '{gerrit-server-name}' trigger-on: - comment-added-contains-event: @@ -156,19 +148,36 @@ - compare-type: 'ANT' pattern: '**' +- job-template: + name: '{project-name}-{stream}-verify-make-{helm_bin}' + + <<: *helm_job_boiler_plate + + scm: + - gerrit-trigger-scm: + refspec: '' + choosing-strategy: 'default' + submodule-recursive: '{submodule-recursive}' + + triggers: + - gerrit: + server-name: '{gerrit-server-name}' + trigger-on: + - comment-added-contains-event: + comment-contains-value: '^Patch Set\s+\d+:\s+helm-latest\s*$' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + file-paths: + - compare-type: 'ANT' + pattern: '**' + builders: - lf-infra-pre-build - - lf-provide-maven-settings: - global-settings-file: '{mvn-global-settings}' - settings-file: '{mvn-settings}' - - shell: | - echo "export HELM_BIN={helm_bin}" > helm.prop + - helm-install: + helm_ver: 3.10.2 - shell: '{obj:pre_build_script}' - shell: '{obj:build_script}' - - oom-create-netrc: - server-id: oom-helm - - inject: - properties-content: | - BUILD_TYPE=release - - shell: '{obj:post_build_script}' - - lf-provide-maven-settings-cleanup diff --git a/jjb/oom/oom-helm.yaml b/jjb/oom/oom-helm.yaml index 40eca918c..6a38faa63 100644 --- a/jjb/oom/oom-helm.yaml +++ b/jjb/oom/oom-helm.yaml @@ -3,23 +3,12 @@ name: oom project-name: oom project: oom + archive-artifacts: "" mvn-settings: oom-settings mvn-global-settings: global-settings - archive-artifacts: "" build-node: ubuntu1804-helm-2c-1g - pre_build_script: !include-raw-escape: shell/helm-repo-init.sh - oom_build_script: | - #!/bin/bash - source helm.prop - set -e -o pipefail - cd kubernetes/ - make HELM_BIN=$HELM_BIN all - oom_build_script_skip_lint: | - #!/bin/bash - source helm.prop - set -e -o pipefail - cd kubernetes/ - make HELM_BIN=$HELM_BIN SKIP_LINT=TRUE all + helm_bin: helm3.8 + pre_build_script: !include-raw-escape: shell/helm/helm-repo-init.sh stream: - "master": @@ -29,23 +18,43 @@ - "jakarta": branch: "jakarta" - helm_bin: - - helm3.8 - jobs: - "{project-name}-{stream}-image-verify": files: "**" - "{project-name}-{stream}-verify-{helm_bin}": - build_script: "{oom_build_script}" - files: "kubernetes/**" + build_script: !include-raw-escape: shell/helm/make-helm-all.sh + files: "**/kubernetes/**" build-timeout: 900 - "{project-name}-{stream}-merge-helm": - build_script: "{oom_build_script}" - post_build_script: !include-raw-escape: shell/publish_helm_charts.sh - files: "kubernetes/**" + build_type: snapshot + build_script: !include-raw-escape: shell/helm/make-helm-all.sh + post_build_script: !include-raw-escape: shell/helm/publish_helm_charts.sh + files: "**/kubernetes/**" build-timeout: 900 - "{project-name}-{stream}-release-helm": - build_script: "{oom_build_script_skip_lint}" - post_build_script: !include-raw-escape: shell/publish_helm_charts.sh - files: "kubernetes/**" + build_type: release + build_script: !include-raw-escape: shell/helm/make-helm-skip-lint.sh + post_build_script: !include-raw-escape: shell/helm/publish_helm_charts.sh + files: "**/kubernetes/**" + build-timeout: 900 + +- project: + name: oom-helm-verify-make-latest + project-name: oom + project: oom + archive-artifacts: "" + build-node: ubuntu1804-helm-2c-1g + helm_bin: helm3.10.2 + pre_build_script: !include-raw-escape: shell/helm/helm-repo-init.sh + + stream: + - "master": + branch: "master" + - "kohn": + branch: "kohn" + + jobs: + - "{project-name}-{stream}-verify-make-{helm_bin}": + build_script: !include-raw-escape: shell/helm/make-helm-skip-lint.sh + files: "**/kubernetes/**" build-timeout: 900 diff --git a/jjb/oom/oom-main-linter.yaml b/jjb/oom/oom-main-linter.yaml index 0f49ead3a..21e670613 100644 --- a/jjb/oom/oom-main-linter.yaml +++ b/jjb/oom/oom-main-linter.yaml @@ -1,3 +1,4 @@ +--- - project: name: oom-main-linters project: oom diff --git a/shell/apply-submodule-patch-oom.sh b/shell/apply-submodule-patch-oom.sh deleted file mode 100644 index ad818fb80..000000000 --- a/shell/apply-submodule-patch-oom.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: EPL-1.0 -############################################################################## -# Copyright (c) 2019 The Linux Foundation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Eclipse Public License v1.0 -# which accompanies this distribution, and is available at -# http://www.eclipse.org/legal/epl-v10.html -############################################################################## - -# Update kubernetes submodule under oom with patch to be verified - -echo '--> apply-submodule-patch-oom.sh' -cd kubernetes/${HELM_MODULE} -remote_path=`git remote -v | grep fetch | awk '{print $2}'` -git fetch ${remote_path} $GERRIT_REFSPEC && git cherry-pick FETCH_HEAD -cd ../.. diff --git a/shell/helm/helm-install.sh b/shell/helm/helm-install.sh new file mode 100755 index 000000000..9fa247772 --- /dev/null +++ b/shell/helm/helm-install.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Ensure we fail the job if any steps fail +set -e -o pipefail + +echo "Installing helm ${HELM_VER}" +mkdir /tmp/helm"${HELM_VER}" +cd /tmp/helm"${HELM_VER}" +wget "https://get.helm.sh/helm-v${HELM_VER}-linux-amd64.tar.gz" +tar xvf helm-v"${HELM_VER}"-linux-amd64.tar.gz +sudo mv linux-amd64/helm /usr/local/bin/helm"${HELM_VER}" +which helm"${HELM_VER}" +cd ~/ +echo "Completed install of helm ${HELM_VER}" + diff --git a/shell/helm-repo-init.sh b/shell/helm/helm-repo-init.sh similarity index 95% rename from shell/helm-repo-init.sh rename to shell/helm/helm-repo-init.sh index 953abb802..f1ef7aaf3 100755 --- a/shell/helm-repo-init.sh +++ b/shell/helm/helm-repo-init.sh @@ -5,7 +5,6 @@ set -e -o pipefail mkdir -p ".chartstorage" chartmuseum --port=6464 --storage="local" --storage-local-rootdir=".chartstorage" &> /dev/null & -source helm.prop $HELM_BIN plugin install --version v0.10.3 https://github.com/chartmuseum/helm-push.git || true $HELM_BIN repo add local http://localhost:6464 $HELM_BIN repo add onap http://localhost:6464 diff --git a/shell/helm/make-helm-all.sh b/shell/helm/make-helm-all.sh new file mode 100755 index 000000000..862f422ac --- /dev/null +++ b/shell/helm/make-helm-all.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# Ensure we fail the job if any steps fail +set -e -o pipefail +cd kubernetes/ +make HELM_BIN=$HELM_BIN all -j2 diff --git a/shell/helm/make-helm-skip-lint.sh b/shell/helm/make-helm-skip-lint.sh new file mode 100755 index 000000000..0a64365d8 --- /dev/null +++ b/shell/helm/make-helm-skip-lint.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# Ensure we fail the job if any steps fail +set -e -o pipefail +cd kubernetes/ +make HELM_BIN=$HELM_BIN SKIP_LINT=TRUE all -j2 diff --git a/shell/oom-create-netrc.sh b/shell/helm/oom-create-netrc.sh similarity index 100% rename from shell/oom-create-netrc.sh rename to shell/helm/oom-create-netrc.sh diff --git a/shell/publish_helm_charts.sh b/shell/helm/publish_helm_charts.sh similarity index 67% rename from shell/publish_helm_charts.sh rename to shell/helm/publish_helm_charts.sh index 4cc27fe2d..460174328 100755 --- a/shell/publish_helm_charts.sh +++ b/shell/helm/publish_helm_charts.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -o pipefail -echo "*** starting releace process for $BUILD_TYPE" +echo "*** starting chart publish process for $BUILD_TYPE" cd kubernetes/dist/packages/ || exit helm_charts=() while IFS= read -a line; do @@ -16,19 +16,15 @@ for chart in "${helm_charts[@]}"; do echo " * snapshot build, pushing to https://nexus3.onap.org/repository/onap-helm-testing/" curl -vn --upload-file "$chart" "https://nexus3.onap.org/repository/onap-helm-testing/" ;; - 'staging') - echo " * staging build, pushing to https://nexus3.onap.org/repository/onap-helm-testing/" - curl -vn --upload-file "$chart" "https://nexus3.onap.org/repository/onap-helm-testing/" - ;; 'release') echo " * release build, pushing to https://nexus3.onap.org/repository/onap-helm-release/" curl -vn --upload-file "$chart" "https://nexus3.onap.org/repository/onap-helm-release/" ;; *) - echo "You must set BUILD_TYPE to one of (snapshot, staging, release)." + echo "You must set BUILD_TYPE to one of (snapshot, release)." exit 1 ;; esac done -echo "*** release process finished" +echo "*** chart publish process finished" cd ../../../ -- 2.16.6