X-Git-Url: https://gerrit.onap.org/r/gitweb?a=blobdiff_plain;f=build%2Fbuild_nexus_blob.sh;h=ddc3145b2a1b6f09b30f4a0bdf3acd508d193db0;hb=cf9b45ba91722e7cb4089cc256af29901893c44a;hp=51ab05e7f127d2791e0ace1c8c67c793b9175353;hpb=30bb2a6332d4e18f0a31e955e9285bf0a3d12411;p=oom%2Foffline-installer.git diff --git a/build/build_nexus_blob.sh b/build/build_nexus_blob.sh index 51ab05e7..ddc3145b 100755 --- a/build/build_nexus_blob.sh +++ b/build/build_nexus_blob.sh @@ -2,7 +2,7 @@ # COPYRIGHT NOTICE STARTS HERE # -# Copyright 2018-2019 © Samsung Electronics Co., Ltd. +# Copyright 2018-2020© Samsung Electronics Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -35,13 +35,13 @@ SCRIPT_LOG="/tmp/$(basename $0)_$(eval ${TIMESTAMP}).log" # Log everything exec &> >(tee -a "${SCRIPT_LOG}") -# Nexus repository location +# Nexus repository properties NEXUS_DOMAIN="nexus" -NEXUS_PORT="8081" -NEXUS_DOCKER_PORT="8082" -NPM_REGISTRY="http://${NEXUS_DOMAIN}:${NEXUS_PORT}/repository/npm-private/" -PYPI_REGISTRY="http://${NEXUS_DOMAIN}:${NEXUS_PORT}/repository/pypi-private/" -DOCKER_REGISTRY="${NEXUS_DOMAIN}:${NEXUS_DOCKER_PORT}" +NEXUS_HOST="127.0.0.1" +NEXUS_EXPOSED_PORT="8081" +NEXUS_PORT=${NEXUS_EXPOSED_PORT} +NEXUS_DOCKER_EXPOSED_PORT="8082" +NEXUS_DOCKER_PORT=${NEXUS_DOCKER_EXPOSED_PORT} DEFAULT_REGISTRY="docker.io" # Nexus repository credentials @@ -54,12 +54,14 @@ LOCAL_PATH="$(readlink -f $(dirname ${0}))" # Defaults DOCKER_LOAD="false" +NPM_PUSH="false" +PYPI_PUSH="false" DATA_DIR="$(realpath ${LOCAL_PATH}/../../resources)" NEXUS_DATA_DIR="${DATA_DIR}/nexus_data" LISTS_DIR="${LOCAL_PATH}/data_lists" # Required dependencies -COMMANDS=(jq docker expect npm twine) +COMMANDS=(jq docker) usage () { echo " @@ -95,38 +97,48 @@ usage () { -o | --output-directory use specific directory for the target blob -p | --pypi use specific list of pypi packages to be pushed into Nexus -rl | --resource-list-directory use specific directory with docker, pypi and npm lists + -c | --container-name use specific Nexus docker container name + -NP | --nexus-port use specific port for published Nexus service + -DP | --docker-port use specific port for published Nexus docker registry port " exit 1 } -publish_ports () { - for REGISTRY in $(sed -n '/\.[^/].*\//p' ${1} | sed -e 's/\/.*$//' | sort -u | grep -v ${DEFAULT_REGISTRY} || true) ${NEXUS_PORT}; do - if [[ ${REGISTRY} != *":"* ]]; then - if [[ ${PUBLISHED_PORTS} != *"80:${NEXUS_DOCKER_PORT}"* ]]; then - PUBLISHED_PORTS="${PUBLISHED_PORTS} -p 80:${NEXUS_DOCKER_PORT}" - fi - else - REGISTRY_PORT="$(sed 's/^.*\:\([[:digit:]]*\)$/\1/' <<< ${REGISTRY})" - if [[ ${PUBLISHED_PORTS} != *"${REGISTRY_PORT}:${NEXUS_DOCKER_PORT}"* ]]; then - PUBLISHED_PORTS="${PUBLISHED_PORTS} -p ${REGISTRY_PORT}:${NEXUS_DOCKER_PORT}" - fi - fi +load_docker_images () { + for ARCHIVE in $(sed $'s/\r// ; /^#/d ; s/\:/\_/g ; s/\//\_/g ; s/$/\.tar/g' ${1} | awk '{ print $1 }'); do + docker load -i ${NXS_SRC_DOCKER_IMG_DIR}/${ARCHIVE} done } -simulated_hosts () { - SIMUL_HOSTS=($(sed -n '/\.[^/].*\//p' ${1} | sed -e 's/\/.*$// ; s/:.*$//' | sort -u | grep -v ${DEFAULT_REGISTRY} || true ) ${NEXUS_DOMAIN}) - for HOST in "${SIMUL_HOSTS[@]}"; do - if ! grep -wq ${HOST} /etc/hosts; then - echo "127.0.0.1 ${HOST}" >> /etc/hosts - fi - done +prepare_npm () { + # Configure NPM registry to our Nexus repository + echo "Configure NPM registry to ${NPM_REGISTRY}" + npm config set registry "${NPM_REGISTRY}" + + # Login to NPM registry + /usr/bin/expect <<- EOF + spawn npm login + expect "Username:" + send "${NEXUS_USERNAME}\n" + expect "Password:" + send "${NEXUS_PASSWORD}\n" + expect Email: + send "${NEXUS_EMAIL}\n" + expect eof + EOF } -load_docker_images () { - for ARCHIVE in $(sed $'s/\r// ; /^#/d ; s/\:/\_/g ; s/\//\_/g ; s/$/\.tar/g' ${1} | awk '{ print $1 }'); do - docker load -i ${NXS_SRC_DOCKER_IMG_DIR}/${ARCHIVE} - done +patch_npm () { + # Patch problematic package + PATCHED_NPM="$(grep tsscmp ${1} | sed $'s/\r// ; s/\\@/\-/ ; s/$/\.tgz/')" + if [[ ! -z "${PATCHED_NPM}" ]] && ! zgrep -aq "${NPM_REGISTRY}" "${PATCHED_NPM}" 2>/dev/null + then + tar xzf "${PATCHED_NPM}" + rm -f "${PATCHED_NPM}" + sed -i 's|\"registry\":\ \".*\"|\"registry\":\ \"'"${NPM_REGISTRY}"'\"|g' package/package.json + tar -zcf "${PATCHED_NPM}" package + rm -rf package + fi } push_npm () { @@ -137,19 +149,15 @@ push_npm () { } push_pip () { - for PACKAGE in $(sed $'s/\r//; s/==/-/' ${NXS_PYPI_LIST}); do - twine upload -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" --repository-url ${PYPI_REGISTRY} ${PACKAGE}* + for PACKAGE in $(sed $'s/\r//; s/==/-/' ${1}); do + twine upload -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" --repository-url ${PYPI_REGISTRY} ${PACKAGE}* > /dev/null echo "PYPI ${PACKAGE} pushed to Nexus" done } docker_login () { - for REGISTRY in $(sed -n '/\.[^/].*\//p' ${1} | sed -e 's/\/.*$//' | sort -u | grep -v ${DEFAULT_REGISTRY}) ${DOCKER_REGISTRY}; do - if ! grep -wqs ${REGISTRY} ~/.docker/config.json; then - echo "Docker login to ${REGISTRY}" - echo -n "${NEXUS_PASSWORD}" | docker login -u "${NEXUS_USERNAME}" --password-stdin ${REGISTRY} > /dev/null - fi - done + echo "Docker login to ${DOCKER_REGISTRY}" + echo -n "${NEXUS_PASSWORD}" | docker --config "${DOCKER_CONFIG_DIR}" login -u "${NEXUS_USERNAME}" --password-stdin ${DOCKER_REGISTRY} > /dev/null } push_docker () { @@ -165,59 +173,90 @@ push_docker () { fi elif [[ -z $(sed -n '/\.[^/].*\//p' <<< ${IMAGE}) ]]; then PUSH="${DOCKER_REGISTRY}/${IMAGE}" - fi - if [[ ! -z ${PUSH} ]]; then - docker tag ${IMAGE} ${PUSH} else - PUSH="${IMAGE}" + # substitute all host names with $DOCKER_REGISTRY + repo_host=$(sed -e 's/\/.*$//' <<< ${IMAGE}) + PUSH="$(sed -e 's/'"${repo_host}"'/'"${DOCKER_REGISTRY}"'/' <<< ${IMAGE})" fi - docker push ${PUSH} + docker tag ${IMAGE} ${PUSH} + docker --config "${DOCKER_CONFIG_DIR}" push ${PUSH} + # Remove created tag + docker rmi ${PUSH} echo "${IMAGE} pushed as ${PUSH} to Nexus" done } -# Verify all dependencies are available in PATH -FAILED_COMMANDS=() -for cmd in ${COMMANDS[*]}; do - command -v $cmd >/dev/null 2>&1 || FAILED_COMMANDS+=($cmd) -done - -if [ ${#FAILED_COMMANDS[*]} -gt 0 ]; then - echo "Following commands where not found in PATH and are required:" - echo ${FAILED_COMMANDS[*]} - echo "Aborting." - exit 1 -fi +validate_container_name () { + # Verify $1 is a valid hostname + if ! echo "${1}" | egrep -q "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$"; + then + echo "ERROR: ${1} is not a valid name!" + exit 1; + fi +} while [ "${1}" != "" ]; do case ${1} in -d | --docker ) shift - NXS_DOCKER_IMG_LISTS+=("${1}") + NXS_DOCKER_IMG_LISTS+=("$(realpath ${1})") ;; -i | --input-directory ) shift - DATA_DIR="${1}" + DATA_DIR="$(realpath ${1})" ;; -ld | --load-docker-images ) DOCKER_LOAD="true" ;; - -n | --npm ) shift - NXS_NPM_LISTS+=("${1}") + -n | --npm ) NPM_PUSH="true" + COMMANDS+=(expect npm) + shift + NXS_NPM_LISTS+=("$(realpath ${1})") + ;; + -c | --container-name ) shift + validate_container_name "${1}" + NEXUS_DOMAIN="${1}" ;; -o | --output-directory ) shift - NEXUS_DATA_DIR="${1}" + NEXUS_DATA_DIR="$(realpath ${1})" ;; - -p | --pypi ) shift - NXS_PYPI_LISTS+=("${1}") + -p | --pypi ) PYPI_PUSH="true" + COMMANDS+=(twine) + shift + NXS_PYPI_LISTS+=("$(realpath ${1})") ;; -rl | --resource-list-directory ) shift - LISTS_DIR="${1}" + LISTS_DIR="$(realpath ${1})" + ;; + -NP | --nexus-port ) shift + NEXUS_PORT="${1}" + ;; + -DP | --docker-port ) shift + NEXUS_DOCKER_PORT="${1}" ;; -h | --help ) usage ;; *) usage + ;; esac shift done +# Verify all dependencies are available in PATH +FAILED_COMMANDS=() +for cmd in ${COMMANDS[*]}; do + command -v $cmd >/dev/null 2>&1 || FAILED_COMMANDS+=($cmd) +done + +if [ ${#FAILED_COMMANDS[*]} -gt 0 ]; then + echo "Following commands where not found in PATH and are required:" + echo ${FAILED_COMMANDS[*]} + echo "Aborting." + exit 1 +fi + +# Nexus repository locations +NPM_REGISTRY="http://${NEXUS_HOST}:${NEXUS_PORT}/repository/npm-private/" +PYPI_REGISTRY="http://${NEXUS_HOST}:${NEXUS_PORT}/repository/pypi-private/" +DOCKER_REGISTRY="${NEXUS_HOST}:${NEXUS_DOCKER_PORT}" + # Setup directories with resources for docker, npm and pypi NXS_SRC_DOCKER_IMG_DIR="${DATA_DIR}/offline_data/docker_images_for_nexus" NXS_SRC_NPM_DIR="${DATA_DIR}/offline_data/npm_tar" @@ -227,39 +266,23 @@ NXS_SRC_PYPI_DIR="${DATA_DIR}/offline_data/pypi" NXS_INFRA_LIST="${LISTS_DIR}/infra_docker_images.list" NXS_DOCKER_IMG_LIST="${LISTS_DIR}/onap_docker_images.list" NXS_RKE_DOCKER_IMG_LIST="${LISTS_DIR}/rke_docker_images.list" -NXS_NPM_LIST="${LISTS_DIR}/onap_npm.list" -NXS_PYPI_LIST="${LISTS_DIR}/onap_pip_packages.list" +NXS_K8S_DOCKER_IMG_LIST="${LISTS_DIR}/k8s_docker_images.list" +NXS_PROMETHEUS_DOCKER_IMG_LIST="${LISTS_DIR}/kube_prometheus_stack_docker_images.list" # Setup Nexus image used for build and install infra NEXUS_IMAGE="$(grep sonatype/nexus3 ${NXS_INFRA_LIST})" NEXUS_IMAGE_TAR="${DATA_DIR}/offline_data/docker_images_infra/$(sed 's/\//\_/ ; s/$/\.tar/ ; s/\:/\_/' <<< ${NEXUS_IMAGE})" # Set default lists if nothing specific defined by user -if [ $((${#NXS_DOCKER_IMG_LISTS[@]} + ${#NXS_NPM_LISTS[@]} + ${#NXS_PYPI_LISTS[@]})) -eq 0 ]; then - NXS_DOCKER_IMG_LISTS=("${NXS_DOCKER_IMG_LIST}" "${NXS_RKE_DOCKER_IMG_LIST}") - NXS_NPM_LISTS[0]="${NXS_NPM_LIST}" - NXS_PYPI_LISTS[0]="${NXS_PYPI_LIST}" +if [ ${#NXS_DOCKER_IMG_LISTS[@]} -eq 0 ]; then + NXS_DOCKER_IMG_LISTS=("${NXS_DOCKER_IMG_LIST}" "${NXS_RKE_DOCKER_IMG_LIST}" "${NXS_K8S_DOCKER_IMG_LIST}" "${NXS_PROMETHEUS_DOCKER_IMG_LIST}") fi -# Backup /etc/hosts -HOSTS_BACKUP="$(eval ${TIMESTAMP}_hosts.bk)" -cp /etc/hosts /etc/${HOSTS_BACKUP} - -# Backup the current docker registry settings -if [ -f ~/.docker/config.json ]; then - DOCKER_CONF_BACKUP="$(eval ${TIMESTAMP}_config.json.bk)" - mv ~/.docker/config.json ~/.docker/${DOCKER_CONF_BACKUP} -fi +# Create Docker client config dir +DOCKER_CONFIG_DIR=$(mktemp -p /tmp -d .docker.XXXXXXXX) # Setup default ports published to host as docker registry -PUBLISHED_PORTS="-p ${NEXUS_PORT}:${NEXUS_PORT} -p ${NEXUS_DOCKER_PORT}:${NEXUS_DOCKER_PORT}" - -# Setup additional ports published to host based on simulated docker registries -# Setup simulated domain names to be able to push all to private Nexus repository -for DOCKER_IMG_LIST in "${NXS_DOCKER_IMG_LISTS[@]}"; do - publish_ports "${DOCKER_IMG_LIST}" - simulated_hosts "${DOCKER_IMG_LIST}" -done +PUBLISHED_PORTS="-p ${NEXUS_PORT}:${NEXUS_EXPOSED_PORT} -p ${NEXUS_DOCKER_PORT}:${NEXUS_DOCKER_EXPOSED_PORT}" # Nexus repository configuration setup NEXUS_CONFIG_GROOVY='import org.sonatype.nexus.security.realm.RealmManager @@ -333,79 +356,64 @@ chown 200:200 ${NEXUS_DATA_DIR} chmod 777 ${NEXUS_DATA_DIR} # Save Nexus version to prevent/catch data incompatibility -docker images --no-trunc | grep sonatype/nexus3 | awk '{ print $1":"$2" "$3}' > ${NEXUS_DATA_DIR}/nexus.ver +# Adding commit informations to have link to data from which the blob was built +cat >> ${NEXUS_DATA_DIR}/nexus.ver << INFO +nexus_image=$(docker image ls ${NEXUS_IMAGE} --no-trunc --format "{{.Repository}}:{{.Tag}}\nnexus_image_digest={{.ID}}") +$(for INDEX in ${!NXS_DOCKER_IMG_LISTS[@]}; do printf 'used_image_list%s=%s\n' "$INDEX" "$(sed 's/^.*\/\(.*\)$/\1/' <<< ${NXS_DOCKER_IMG_LISTS[$INDEX]})"; done) +$(sed -n 's/^.*OOM\ commit\ /oom_repo_commit=/p' ${NXS_DOCKER_IMG_LISTS[@]}) +installer_repo_commit=$(git --git-dir="${LOCAL_PATH}/../.git" rev-parse HEAD) +INFO # Start the Nexus NEXUS_CONT_ID=$(docker run -d --rm -v ${NEXUS_DATA_DIR}:/nexus-data:rw --name ${NEXUS_DOMAIN} ${PUBLISHED_PORTS} ${NEXUS_IMAGE}) echo "Waiting for Nexus to fully start" -until curl -su ${NEXUS_USERNAME}:${NEXUS_PASSWORD} http://${NEXUS_DOMAIN}:${NEXUS_PORT}/service/metrics/healthcheck | grep '"healthy":true' > /dev/null ; do +until curl -su ${NEXUS_USERNAME}:${NEXUS_PASSWORD} http://${NEXUS_HOST}:${NEXUS_PORT}/service/metrics/healthcheck | grep '"healthy":true' > /dev/null ; do printf "." sleep 3 done echo -e "\nNexus started" # Configure the nexus repository -curl -sX POST --header 'Content-Type: application/json' --data-binary "${NEXUS_CONFIG}" http://${NEXUS_USERNAME}:${NEXUS_PASSWORD}@${NEXUS_DOMAIN}:${NEXUS_PORT}/service/rest/v1/script -curl -sX POST --header "Content-Type: text/plain" http://${NEXUS_USERNAME}:${NEXUS_PASSWORD}@${NEXUS_DOMAIN}:${NEXUS_PORT}/service/rest/v1/script/configure/run > /dev/null +curl -sX POST --header 'Content-Type: application/json' --data-binary "${NEXUS_CONFIG}" http://${NEXUS_USERNAME}:${NEXUS_PASSWORD}@${NEXUS_HOST}:${NEXUS_PORT}/service/rest/v1/script +curl -sX POST --header "Content-Type: text/plain" http://${NEXUS_USERNAME}:${NEXUS_PASSWORD}@${NEXUS_HOST}:${NEXUS_PORT}/service/rest/v1/script/configure/run > /dev/null ########################### # Populate NPM repository # ########################### - -# Configure NPM registry to our Nexus repository -echo "Configure NPM registry to ${NPM_REGISTRY}" -npm config set registry "${NPM_REGISTRY}" - -# Login to NPM registry -/usr/bin/expect </dev/null; then - tar xzf "${PATCHED_NPM}" - rm -f "${PATCHED_NPM}" - sed -i 's|\"registry\":\ \".*\"|\"registry\":\ \"'"${NPM_REGISTRY}"'\"|g' package/package.json - tar -zcf "${PATCHED_NPM}" package - rm -rf package +if [ $NPM_PUSH == "true" ]; then + prepare_npm + pushd ${NXS_SRC_NPM_DIR} + for NPM_LIST in "${NXS_NPM_LISTS[@]}"; do + patch_npm "${NPM_LIST}" + push_npm "${NPM_LIST}" + done + popd + # Return default settings + npm logout + npm config set registry "https://registry.npmjs.org" fi -# Push NPM packages to Nexus repository -for NPM_LIST in "${NXS_NPM_LISTS[@]}"; do - push_npm "${NPM_LIST}" -done -popd -npm logout - ############################### ## Populate PyPi repository # ############################### - -pushd ${NXS_SRC_PYPI_DIR} -for PYPI_LIST in "${NXS_PYPI_LISTS[@]}"; do - push_pip "${PYPI_LIST}" -done -popd +if [ $PYPI_PUSH == "true" ]; then + pushd ${NXS_SRC_PYPI_DIR} + for PYPI_LIST in "${NXS_PYPI_LISTS[@]}"; do + push_pip "${PYPI_LIST}" + done + popd +fi ############################### ## Populate Docker repository # ############################### -# Login to simulated docker registries -# Push images to private nexus based on the list -# Images from default registry need to be tagged to private registry -# and those without defined repository in tag uses default repository 'library' +# Login to docker registry simulated by Nexus container +# Push images to private nexus based on the lists +# All images need to be tagged to simulated registry +# and those without defined repository in tag use default repository 'library' +docker_login for DOCKER_IMG_LIST in "${NXS_DOCKER_IMG_LISTS[@]}"; do - docker_login "${DOCKER_IMG_LIST}" push_docker "${DOCKER_IMG_LIST}" done @@ -413,20 +421,13 @@ done # Stop the Nexus and cleanup # ############################## -echo "Stopping Nexus and returning backups" +echo "Stopping Nexus" # Stop the Nexus docker stop ${NEXUS_CONT_ID} > /dev/null -# Return backed up configuration files -mv -f "/etc/${HOSTS_BACKUP}" /etc/hosts - -if [ -f ~/.docker/${DOCKER_CONF_BACKUP} ]; then - mv -f ~/.docker/${DOCKER_CONF_BACKUP} ~/.docker/config.json -fi - -# Return default settings -npm config set registry "https://registry.npmjs.org" +# Drop temporary Docker client config dir +rm -rf ${DOCKER_CONFIG_DIR} echo "Nexus blob is built" exit 0