Configure PyPI releases

This changes adds the following functionality for PyPI releases

- macros
 - A PyPI server macro

- scripts
 - A script to build artifacts
 - A script to publish artifacts

- templates
 - Templates for release

- jobs
 - Jobs for release
   - Nexus3 staging
   - Nexus3 release
   - PyPI index will be added at a later date

Issue-ID: CIMAN-137
Change-Id: I53522ed2fa2f462afebaac94c44fa2a9979e9c2a
Signed-off-by: Vanessa Rene Valderrama <vvalderrama@linuxfoundation.org>
Signed-off-by: Jessica Wagantall <jwagantall@linuxfoundation.org>
Signed-off-by: Jeremy Phelps <jphelps@linuxfoundation.org>

diff --git a/jjb/dcaegen2/dcaegen2-utils.yaml b/jjb/dcaegen2/dcaegen2-utils.yaml
index f91a4c2..d6dcb01 100644 (file)
--- a/jjb/dcaegen2/dcaegen2-utils.yaml
+++ b/jjb/dcaegen2/dcaegen2-utils.yaml
@@ -2,11 +2,31 @@
 - project:
     name: dcaegen2-utils
     project-name: 'dcaegen2-utils'
+    python-version: python3
+    subproject:
+      - 'dcaeapplib':
+          tox-dir: dcaeapplib/
+          tox-envs: ''
+      - 'onap-dcae-dbs-docker-client':
+          tox-dir: onap-dcae-dbs-docker-client/
+          tox-envs: ''
+      - 'onap-dcae-dcaepolicy-lib':
+          tox-dir: onap-dcae-dcaepolicy-lib/
+          tox-envs: ''
+      - 'python-discovery-client':
+          tox-dir: python-discovery-client/
+          tox-envs: ''
+      - 'python-dockering':
+          tox-dir: python-dockering/
+          tox-envs: ''
+
     jobs:
       - gerrit-maven-clm
       - '{project-name}-{stream}-verify-java'
       - '{project-name}-{stream}-merge-java'
       - '{project-name}-{stream}-release-version-java-daily'
+      - '{project-name}-{subproject}-python-staging-{stream}'
+      - '{project-name}-{subproject}-python-release-{stream}'
 
     project: 'dcaegen2/utils'
     stream:

diff --git a/jjb/global-macros.yaml b/jjb/global-macros.yaml
index 826b604..85c2756 100644 (file)
--- a/jjb/global-macros.yaml
+++ b/jjb/global-macros.yaml
@@ -66,6 +66,14 @@
     builders:
       - shell: !include-raw: include-docker-login.sh
 
+- builder:
+    name: publish-pypi
+    builders:
+      - inject:
+          properties-content: PYPI_SERVER={pypi-server}
+      - shell:
+          !include-raw-escape: ../shell/pypi-publish.sh
+
 # PARAMETERS
 - parameter:
     name: infra-parameters

diff --git a/jjb/onap-jjb/onap-pypi-template.yaml b/jjb/onap-jjb/onap-pypi-template.yaml
new file mode 100644 (file)
index 0000000..bfd7099
--- /dev/null
+++ b/jjb/onap-jjb/onap-pypi-template.yaml
@@ -0,0 +1,194 @@
+---
+# Job definition Anchors
+- pypi_job_boiler_plate: &pypi_job_boiler_plate
+    name: pypi_job_boiler_plate
+
+    #####################
+    #   Job Variables   #
+    #####################
+
+    gerrit_merge_triggers:
+      - change-merged-event
+      - comment-added-contains-event:
+          comment-contains-value: pypi-remerge$
+
+    gerrit_release_trigger:
+      - comment-added-contains-event:
+          comment-contains-value: pypi-release$
+
+    gerrit_trigger_file_paths:
+      - compare-type: ANT
+        pattern: '**'
+
+    #####################
+    # Job Configuration #
+    #####################
+
+    project-type: freestyle
+    node: '{build-node}'
+
+    parameters:
+      - lf-infra-parameters:
+          project: '{project}'
+          branch: '{branch}'
+          stream: '{stream}'
+          lftools-version: '{lftools-version}'
+      - lf-infra-tox-parameters:
+          tox-dir: '{tox-dir}'
+          tox-envs: '{tox-envs}'
+
+
+    properties:
+      - lf-infra-properties:
+          build-days-to-keep: '{build-days-to-keep}'
+
+    scm:
+      - lf-infra-gerrit-scm:
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+          git-url: '$GIT_URL/$PROJECT'
+          refspec: '$GERRIT_REFSPEC'
+          branch: '$GERRIT_BRANCH'
+          submodule-recursive: true
+          choosing-strategy: default
+
+    wrappers:
+      - lf-infra-wrappers:
+          build-timeout: '{build-timeout}'
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+
+    publishers:
+      - lf-infra-publish
+
+- job-template:
+    name: '{project-name}-python-staging-{stream}'
+
+    # Job template for PyPI staging jobs
+    # Daily Builds
+
+    <<: *pypi_job_boiler_plate
+
+    triggers:
+      - gerrit:
+          server-name: '{gerrit-server-name}'
+          trigger-on: '{obj:gerrit_merge_triggers}'
+          projects:
+            - project-compare-type: ANT
+              project-pattern: '{project}'
+              branches:
+                - branch-compare-type: ANT
+                  branch-pattern: '**/{branch}'
+              file-paths: '{obj:gerrit_trigger_file_paths}'
+
+    builders:
+      - lf-infra-tox-install:
+          python-version: '{python-version}'
+      - shell: !include-raw-escape: ../global-jjb/shell/tox-run.sh
+      - config-file-provider:
+          files:
+            - file-id: pypirc
+              target: '$HOME/.pypirc'
+      - shell:
+          !include-raw-escape: ../shell/pypi-dist-build.sh
+
+      - publish-pypi:
+          pypi-server: staging
+
+- job-template:
+    name: '{project-name}-python-release-{stream}'
+
+    # Job template for PyPI release jobs
+
+    <<: *pypi_job_boiler_plate
+
+    triggers:
+      - gerrit:
+          server-name: '{gerrit-server-name}'
+          trigger-on: '{obj:gerrit_release_trigger}'
+          projects:
+            - project-compare-type: ANT
+              project-pattern: '{project}'
+              branches:
+                - branch-compare-type: ANT
+                  branch-pattern: '**/{branch}'
+              file-paths: '{obj:gerrit_trigger_file_paths}'
+
+    builders:
+      - lf-infra-tox-install:
+          python-version: '{python-version}'
+      - shell: !include-raw-escape: ../global-jjb/shell/tox-run.sh
+      - config-file-provider:
+          files:
+            - file-id: pypirc
+              target: '$HOME/.pypirc'
+      - shell:
+          !include-raw-escape: ../shell/pypi-dist-build.sh
+
+      - publish-pypi:
+          pypi-server: release
+
+- job-template:
+    name: '{project-name}-{subproject}-python-staging-{stream}'
+
+    # Job template for PyPI staging jobs
+    # Daily Builds
+
+    <<: *pypi_job_boiler_plate
+
+    triggers:
+      - gerrit:
+          server-name: '{gerrit-server-name}'
+          trigger-on: '{obj:gerrit_merge_triggers}'
+          projects:
+            - project-compare-type: ANT
+              project-pattern: '{project}'
+              branches:
+                - branch-compare-type: ANT
+                  branch-pattern: '**/{branch}'
+              file-paths: '{obj:gerrit_trigger_file_paths}'
+
+    builders:
+      - lf-infra-tox-install:
+          python-version: '{python-version}'
+      - shell: !include-raw-escape: ../global-jjb/shell/tox-run.sh
+      - config-file-provider:
+          files:
+            - file-id: pypirc
+              target: '$HOME/.pypirc'
+      - shell:
+          !include-raw-escape: ../shell/pypi-dist-build.sh
+
+      - publish-pypi:
+          pypi-server: staging
+
+- job-template:
+    name: '{project-name}-{subproject}-python-release-{stream}'
+
+    # Job template for PyPI release jobs
+
+    <<: *pypi_job_boiler_plate
+
+    triggers:
+      - gerrit:
+          server-name: '{gerrit-server-name}'
+          trigger-on: '{obj:gerrit_release_trigger}'
+          projects:
+            - project-compare-type: ANT
+              project-pattern: '{project}'
+              branches:
+                - branch-compare-type: ANT
+                  branch-pattern: '**/{branch}'
+              file-paths: '{obj:gerrit_trigger_file_paths}'
+
+    builders:
+      - lf-infra-tox-install:
+          python-version: '{python-version}'
+      - shell: !include-raw-escape: ../global-jjb/shell/tox-run.sh
+      - config-file-provider:
+          files:
+            - file-id: pypirc
+              target: '$HOME/.pypirc'
+      - shell:
+          !include-raw-escape: ../shell/pypi-dist-build.sh
+
+      - publish-pypi:
+          pypi-server: release

diff --git a/shell/pypi-dist-build.sh b/shell/pypi-dist-build.sh
new file mode 100644 (file)
index 0000000..7733f62
--- /dev/null
+++ b/shell/pypi-dist-build.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+# Script to build PyPI artifacts
+set -e -x -o pipefail
+
+virtualenv /tmp/v/twine
+source "/tmp/v/twine/bin/activate"
+
+pip install twine wheel
+
+cd "$WORKSPACE/$TOX_DIR"
+python setup.py sdist bdist_wheel

diff --git a/shell/pypi-get.sh b/shell/pypi-get.sh
new file mode 100644 (file)
index 0000000..40900a5
--- /dev/null
+++ b/shell/pypi-get.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Script to download PyPI artifacts
+
+PROJECT_NAME=$(grep name= setup.py | cut -d"'" -f2)
+ARTIFACT_VERSION=$(grep __version__ ${PROJECT_NAME}/_version.py | cut -d'"' -f2)
+REPO_URL="https://nexus3.onap.org/repository/PyPi.staging/packages"
+TAR_NAME="$REPO_URL/$PROJECT_NAME/$ARTIFACT_VERSION/$PROJECT_NAME-$ARTIFACT_VERSION.tar.gz"
+WHEEL_NAME="${REPO_URL}/${PROJECT_NAME}/${ARTIFACT_VERSION}/${PROJECT_NAME}-${ARTIFACT_VERSION}-py2-none-any.whl"
+
+mkdir dist
+cd dist
+
+wget ${TAR_NAME}
+wget ${WHEEL_NAME}

diff --git a/shell/pypi-publish.sh b/shell/pypi-publish.sh
new file mode 100644 (file)
index 0000000..7ea3fa2
--- /dev/null
+++ b/shell/pypi-publish.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+# Script to publush PyPI artifacts
+set -e -x -o pipefail
+
+virtualenv /tmp/v/twine
+source "/tmp/v/twine/bin/activate"
+
+pip install twine
+
+cd "$WORKSPACE/$TOX_DIR"
+twine upload -r $PYPI_SERVER dist/*