# COPYRIGHT NOTICE STARTS HERE
#
-# Copyright 2018-2019 © Samsung Electronics Co., Ltd.
+# Copyright 2018-2020© Samsung Electronics Co., Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# Log everything
exec &> >(tee -a "${SCRIPT_LOG}")
-# Nexus repository location
+# Nexus repository properties
NEXUS_DOMAIN="nexus"
-NEXUS_PORT="8081"
-NEXUS_DOCKER_PORT="8082"
-NPM_REGISTRY="http://${NEXUS_DOMAIN}:${NEXUS_PORT}/repository/npm-private/"
-PYPI_REGISTRY="http://${NEXUS_DOMAIN}:${NEXUS_PORT}/repository/pypi-private/"
-DOCKER_REGISTRY="${NEXUS_DOMAIN}:${NEXUS_DOCKER_PORT}"
+NEXUS_HOST="127.0.0.1"
+NEXUS_EXPOSED_PORT="8081"
+NEXUS_PORT=${NEXUS_EXPOSED_PORT}
+NEXUS_DOCKER_EXPOSED_PORT="8082"
+NEXUS_DOCKER_PORT=${NEXUS_DOCKER_EXPOSED_PORT}
DEFAULT_REGISTRY="docker.io"
# Nexus repository credentials
# Defaults
DOCKER_LOAD="false"
+NPM_PUSH="false"
+PYPI_PUSH="false"
DATA_DIR="$(realpath ${LOCAL_PATH}/../../resources)"
NEXUS_DATA_DIR="${DATA_DIR}/nexus_data"
LISTS_DIR="${LOCAL_PATH}/data_lists"
# Required dependencies
-COMMANDS=(jq docker expect npm twine)
+COMMANDS=(jq docker)
usage () {
echo "
-o | --output-directory use specific directory for the target blob
-p | --pypi use specific list of pypi packages to be pushed into Nexus
-rl | --resource-list-directory use specific directory with docker, pypi and npm lists
+ -c | --container-name use specific Nexus docker container name
+ -NP | --nexus-port use specific port for published Nexus service
+ -DP | --docker-port use specific port for published Nexus docker registry port
"
exit 1
}
-publish_ports () {
- for REGISTRY in $(sed -n '/\.[^/].*\//p' ${1} | sed -e 's/\/.*$//' | sort -u | grep -v ${DEFAULT_REGISTRY} || true) ${NEXUS_PORT}; do
- if [[ ${REGISTRY} != *":"* ]]; then
- if [[ ${PUBLISHED_PORTS} != *"80:${NEXUS_DOCKER_PORT}"* ]]; then
- PUBLISHED_PORTS="${PUBLISHED_PORTS} -p 80:${NEXUS_DOCKER_PORT}"
- fi
- else
- REGISTRY_PORT="$(sed 's/^.*\:\([[:digit:]]*\)$/\1/' <<< ${REGISTRY})"
- if [[ ${PUBLISHED_PORTS} != *"${REGISTRY_PORT}:${NEXUS_DOCKER_PORT}"* ]]; then
- PUBLISHED_PORTS="${PUBLISHED_PORTS} -p ${REGISTRY_PORT}:${NEXUS_DOCKER_PORT}"
- fi
- fi
+load_docker_images () {
+ for ARCHIVE in $(sed $'s/\r// ; /^#/d ; s/\:/\_/g ; s/\//\_/g ; s/$/\.tar/g' ${1} | awk '{ print $1 }'); do
+ docker load -i ${NXS_SRC_DOCKER_IMG_DIR}/${ARCHIVE}
done
}
-simulated_hosts () {
- SIMUL_HOSTS=($(sed -n '/\.[^/].*\//p' ${1} | sed -e 's/\/.*$// ; s/:.*$//' | sort -u | grep -v ${DEFAULT_REGISTRY} || true ) ${NEXUS_DOMAIN})
- for HOST in "${SIMUL_HOSTS[@]}"; do
- if ! grep -wq ${HOST} /etc/hosts; then
- echo "127.0.0.1 ${HOST}" >> /etc/hosts
- fi
- done
+prepare_npm () {
+ # Configure NPM registry to our Nexus repository
+ echo "Configure NPM registry to ${NPM_REGISTRY}"
+ npm config set registry "${NPM_REGISTRY}"
+
+ # Login to NPM registry
+ /usr/bin/expect <<- EOF
+ spawn npm login
+ expect "Username:"
+ send "${NEXUS_USERNAME}\n"
+ expect "Password:"
+ send "${NEXUS_PASSWORD}\n"
+ expect Email:
+ send "${NEXUS_EMAIL}\n"
+ expect eof
+ EOF
}
-load_docker_images () {
- for ARCHIVE in $(sed $'s/\r// ; /^#/d ; s/\:/\_/g ; s/\//\_/g ; s/$/\.tar/g' ${1} | awk '{ print $1 }'); do
- docker load -i ${NXS_SRC_DOCKER_IMG_DIR}/${ARCHIVE}
- done
+patch_npm () {
+ # Patch problematic package
+ PATCHED_NPM="$(grep tsscmp ${1} | sed $'s/\r// ; s/\\@/\-/ ; s/$/\.tgz/')"
+ if [[ ! -z "${PATCHED_NPM}" ]] && ! zgrep -aq "${NPM_REGISTRY}" "${PATCHED_NPM}" 2>/dev/null
+ then
+ tar xzf "${PATCHED_NPM}"
+ rm -f "${PATCHED_NPM}"
+ sed -i 's|\"registry\":\ \".*\"|\"registry\":\ \"'"${NPM_REGISTRY}"'\"|g' package/package.json
+ tar -zcf "${PATCHED_NPM}" package
+ rm -rf package
+ fi
}
push_npm () {
}
push_pip () {
- for PACKAGE in $(sed $'s/\r//; s/==/-/' ${NXS_PYPI_LIST}); do
- twine upload -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" --repository-url ${PYPI_REGISTRY} ${PACKAGE}*
+ for PACKAGE in $(sed $'s/\r//; s/==/-/' ${1}); do
+ twine upload -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" --repository-url ${PYPI_REGISTRY} ${PACKAGE}* > /dev/null
echo "PYPI ${PACKAGE} pushed to Nexus"
done
}
docker_login () {
- for REGISTRY in $(sed -n '/\.[^/].*\//p' ${1} | sed -e 's/\/.*$//' | sort -u | grep -v ${DEFAULT_REGISTRY}) ${DOCKER_REGISTRY}; do
- if ! grep -wq ${REGISTRY} ~/.docker/config.json; then
- echo "Docker login to ${REGISTRY}"
- echo -n "${NEXUS_PASSWORD}" | docker login -u "${NEXUS_USERNAME}" --password-stdin ${REGISTRY} > /dev/null
- fi
- done
+ echo "Docker login to ${DOCKER_REGISTRY}"
+ echo -n "${NEXUS_PASSWORD}" | docker --config "${DOCKER_CONFIG_DIR}" login -u "${NEXUS_USERNAME}" --password-stdin ${DOCKER_REGISTRY} > /dev/null
}
push_docker () {
fi
elif [[ -z $(sed -n '/\.[^/].*\//p' <<< ${IMAGE}) ]]; then
PUSH="${DOCKER_REGISTRY}/${IMAGE}"
- fi
- if [[ ! -z ${PUSH} ]]; then
- docker tag ${IMAGE} ${PUSH}
else
- PUSH="${IMAGE}"
+ # substitute all host names with $DOCKER_REGISTRY
+ repo_host=$(sed -e 's/\/.*$//' <<< ${IMAGE})
+ PUSH="$(sed -e 's/'"${repo_host}"'/'"${DOCKER_REGISTRY}"'/' <<< ${IMAGE})"
fi
- docker push ${PUSH}
+ docker tag ${IMAGE} ${PUSH}
+ docker --config "${DOCKER_CONFIG_DIR}" push ${PUSH}
+ # Remove created tag
+ docker rmi ${PUSH}
echo "${IMAGE} pushed as ${PUSH} to Nexus"
done
}
-# Verify all dependencies are available in PATH
-FAILED_COMMANDS=()
-for cmd in ${COMMANDS[*]}; do
- command -v $cmd >/dev/null 2>&1 || FAILED_COMMANDS+=($cmd)
-done
-
-if [ ${#FAILED_COMMANDS[*]} -gt 0 ]; then
- echo "Following commands where not found in PATH and are required:"
- echo ${FAILED_COMMANDS[*]}
- echo "Aborting."
- exit 1
-fi
+validate_container_name () {
+ # Verify $1 is a valid hostname
+ if ! echo "${1}" | egrep -q "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$";
+ then
+ echo "ERROR: ${1} is not a valid name!"
+ exit 1;
+ fi
+}
while [ "${1}" != "" ]; do
case ${1} in
-d | --docker ) shift
- NXS_DOCKER_IMG_LISTS+=("${1}")
+ NXS_DOCKER_IMG_LISTS+=("$(realpath ${1})")
;;
-i | --input-directory ) shift
- DATA_DIR="${1}"
+ DATA_DIR="$(realpath ${1})"
;;
-ld | --load-docker-images ) DOCKER_LOAD="true"
;;
- -n | --npm ) shift
- NXS_NPM_LISTS+=("${1}")
+ -n | --npm ) NPM_PUSH="true"
+ COMMANDS+=(expect npm)
+ shift
+ NXS_NPM_LISTS+=("$(realpath ${1})")
+ ;;
+ -c | --container-name ) shift
+ validate_container_name "${1}"
+ NEXUS_DOMAIN="${1}"
;;
-o | --output-directory ) shift
- NEXUS_DATA_DIR="${1}"
+ NEXUS_DATA_DIR="$(realpath ${1})"
;;
- -p | --pypi ) shift
- NXS_PYPI_LISTS+=("${1}")
+ -p | --pypi ) PYPI_PUSH="true"
+ COMMANDS+=(twine)
+ shift
+ NXS_PYPI_LISTS+=("$(realpath ${1})")
;;
-rl | --resource-list-directory ) shift
- LISTS_DIR="${1}"
+ LISTS_DIR="$(realpath ${1})"
+ ;;
+ -NP | --nexus-port ) shift
+ NEXUS_PORT="${1}"
+ ;;
+ -DP | --docker-port ) shift
+ NEXUS_DOCKER_PORT="${1}"
;;
-h | --help ) usage
;;
*) usage
+ ;;
esac
shift
done
+# Verify all dependencies are available in PATH
+FAILED_COMMANDS=()
+for cmd in ${COMMANDS[*]}; do
+ command -v $cmd >/dev/null 2>&1 || FAILED_COMMANDS+=($cmd)
+done
+
+if [ ${#FAILED_COMMANDS[*]} -gt 0 ]; then
+ echo "Following commands where not found in PATH and are required:"
+ echo ${FAILED_COMMANDS[*]}
+ echo "Aborting."
+ exit 1
+fi
+
+# Nexus repository locations
+NPM_REGISTRY="http://${NEXUS_HOST}:${NEXUS_PORT}/repository/npm-private/"
+PYPI_REGISTRY="http://${NEXUS_HOST}:${NEXUS_PORT}/repository/pypi-private/"
+DOCKER_REGISTRY="${NEXUS_HOST}:${NEXUS_DOCKER_PORT}"
+
# Setup directories with resources for docker, npm and pypi
NXS_SRC_DOCKER_IMG_DIR="${DATA_DIR}/offline_data/docker_images_for_nexus"
NXS_SRC_NPM_DIR="${DATA_DIR}/offline_data/npm_tar"
NXS_INFRA_LIST="${LISTS_DIR}/infra_docker_images.list"
NXS_DOCKER_IMG_LIST="${LISTS_DIR}/onap_docker_images.list"
NXS_RKE_DOCKER_IMG_LIST="${LISTS_DIR}/rke_docker_images.list"
-NXS_NPM_LIST="${LISTS_DIR}/onap_npm.list"
-NXS_PYPI_LIST="${LISTS_DIR}/onap_pip_packages.list"
+NXS_K8S_DOCKER_IMG_LIST="${LISTS_DIR}/k8s_docker_images.list"
+NXS_PROMETHEUS_DOCKER_IMG_LIST="${LISTS_DIR}/kube_prometheus_stack_docker_images.list"
# Setup Nexus image used for build and install infra
NEXUS_IMAGE="$(grep sonatype/nexus3 ${NXS_INFRA_LIST})"
NEXUS_IMAGE_TAR="${DATA_DIR}/offline_data/docker_images_infra/$(sed 's/\//\_/ ; s/$/\.tar/ ; s/\:/\_/' <<< ${NEXUS_IMAGE})"
# Set default lists if nothing specific defined by user
-if [ $((${#NXS_DOCKER_IMG_LISTS[@]} + ${#NXS_NPM_LISTS[@]} + ${#NXS_PYPI_LISTS[@]})) -eq 0 ]; then
- NXS_DOCKER_IMG_LISTS=("${NXS_DOCKER_IMG_LIST}" "${NXS_RKE_DOCKER_IMG_LIST}")
- NXS_NPM_LISTS[0]="${NXS_NPM_LIST}"
- NXS_PYPI_LISTS[0]="${NXS_PYPI_LIST}"
+if [ ${#NXS_DOCKER_IMG_LISTS[@]} -eq 0 ]; then
+ NXS_DOCKER_IMG_LISTS=("${NXS_DOCKER_IMG_LIST}" "${NXS_RKE_DOCKER_IMG_LIST}" "${NXS_K8S_DOCKER_IMG_LIST}" "${NXS_PROMETHEUS_DOCKER_IMG_LIST}")
fi
-# Backup /etc/hosts
-HOSTS_BACKUP="$(eval ${TIMESTAMP}_hosts.bk)"
-cp /etc/hosts /etc/${HOSTS_BACKUP}
-
-# Backup the current docker registry settings
-if [ -f ~/.docker/config.json ]; then
- DOCKER_CONF_BACKUP="$(eval ${TIMESTAMP}_config.json.bk)"
- mv ~/.docker/config.json ~/.docker/${DOCKER_CONF_BACKUP}
-fi
+# Create Docker client config dir
+DOCKER_CONFIG_DIR=$(mktemp -p /tmp -d .docker.XXXXXXXX)
# Setup default ports published to host as docker registry
-PUBLISHED_PORTS="-p ${NEXUS_PORT}:${NEXUS_PORT} -p ${NEXUS_DOCKER_PORT}:${NEXUS_DOCKER_PORT}"
-
-# Setup additional ports published to host based on simulated docker registries
-# Setup simulated domain names to be able to push all to private Nexus repository
-for DOCKER_IMG_LIST in "${NXS_DOCKER_IMG_LISTS[@]}"; do
- publish_ports "${DOCKER_IMG_LIST}"
- simulated_hosts "${DOCKER_IMG_LIST}"
-done
+PUBLISHED_PORTS="-p ${NEXUS_PORT}:${NEXUS_EXPOSED_PORT} -p ${NEXUS_DOCKER_PORT}:${NEXUS_DOCKER_EXPOSED_PORT}"
# Nexus repository configuration setup
NEXUS_CONFIG_GROOVY='import org.sonatype.nexus.security.realm.RealmManager
chmod 777 ${NEXUS_DATA_DIR}
# Save Nexus version to prevent/catch data incompatibility
-docker images --no-trunc | grep sonatype/nexus3 | awk '{ print $1":"$2" "$3}' > ${NEXUS_DATA_DIR}/nexus.ver
+# Adding commit informations to have link to data from which the blob was built
+cat >> ${NEXUS_DATA_DIR}/nexus.ver << INFO
+nexus_image=$(docker image ls ${NEXUS_IMAGE} --no-trunc --format "{{.Repository}}:{{.Tag}}\nnexus_image_digest={{.ID}}")
+$(for INDEX in ${!NXS_DOCKER_IMG_LISTS[@]}; do printf 'used_image_list%s=%s\n' "$INDEX" "$(sed 's/^.*\/\(.*\)$/\1/' <<< ${NXS_DOCKER_IMG_LISTS[$INDEX]})"; done)
+$(sed -n 's/^.*OOM\ commit\ /oom_repo_commit=/p' ${NXS_DOCKER_IMG_LISTS[@]})
+installer_repo_commit=$(git --git-dir="${LOCAL_PATH}/../.git" rev-parse HEAD)
+INFO
# Start the Nexus
NEXUS_CONT_ID=$(docker run -d --rm -v ${NEXUS_DATA_DIR}:/nexus-data:rw --name ${NEXUS_DOMAIN} ${PUBLISHED_PORTS} ${NEXUS_IMAGE})
echo "Waiting for Nexus to fully start"
-until curl -su ${NEXUS_USERNAME}:${NEXUS_PASSWORD} http://${NEXUS_DOMAIN}:${NEXUS_PORT}/service/metrics/healthcheck | grep '"healthy":true' > /dev/null ; do
+until curl -su ${NEXUS_USERNAME}:${NEXUS_PASSWORD} http://${NEXUS_HOST}:${NEXUS_PORT}/service/metrics/healthcheck | grep '"healthy":true' > /dev/null ; do
printf "."
sleep 3
done
echo -e "\nNexus started"
# Configure the nexus repository
-curl -sX POST --header 'Content-Type: application/json' --data-binary "${NEXUS_CONFIG}" http://${NEXUS_USERNAME}:${NEXUS_PASSWORD}@${NEXUS_DOMAIN}:${NEXUS_PORT}/service/rest/v1/script
-curl -sX POST --header "Content-Type: text/plain" http://${NEXUS_USERNAME}:${NEXUS_PASSWORD}@${NEXUS_DOMAIN}:${NEXUS_PORT}/service/rest/v1/script/configure/run > /dev/null
+curl -sX POST --header 'Content-Type: application/json' --data-binary "${NEXUS_CONFIG}" http://${NEXUS_USERNAME}:${NEXUS_PASSWORD}@${NEXUS_HOST}:${NEXUS_PORT}/service/rest/v1/script
+curl -sX POST --header "Content-Type: text/plain" http://${NEXUS_USERNAME}:${NEXUS_PASSWORD}@${NEXUS_HOST}:${NEXUS_PORT}/service/rest/v1/script/configure/run > /dev/null
###########################
# Populate NPM repository #
###########################
-
-# Configure NPM registry to our Nexus repository
-echo "Configure NPM registry to ${NPM_REGISTRY}"
-npm config set registry "${NPM_REGISTRY}"
-
-# Login to NPM registry
-/usr/bin/expect <<EOF
-spawn npm login
-expect "Username:"
-send "${NEXUS_USERNAME}\n"
-expect "Password:"
-send "${NEXUS_PASSWORD}\n"
-expect Email:
-send "${NEXUS_EMAIL}\n"
-expect eof
-EOF
-
-# Patch problematic package
-pushd ${NXS_SRC_NPM_DIR}
-PATCHED_NPM="$(grep tsscmp ${NXS_NPM_LIST} | sed $'s/\r// ; s/\\@/\-/ ; s/$/\.tgz/')"
-if [[ ! -z "${PATCHED_NPM}" ]] && ! zgrep -aq "${NPM_REGISTRY}" "${PATCHED_NPM}" 2>/dev/null; then
- tar xzf "${PATCHED_NPM}"
- rm -f "${PATCHED_NPM}"
- sed -i 's|\"registry\":\ \".*\"|\"registry\":\ \"'"${NPM_REGISTRY}"'\"|g' package/package.json
- tar -zcf "${PATCHED_NPM}" package
- rm -rf package
+if [ $NPM_PUSH == "true" ]; then
+ prepare_npm
+ pushd ${NXS_SRC_NPM_DIR}
+ for NPM_LIST in "${NXS_NPM_LISTS[@]}"; do
+ patch_npm "${NPM_LIST}"
+ push_npm "${NPM_LIST}"
+ done
+ popd
+ # Return default settings
+ npm logout
+ npm config set registry "https://registry.npmjs.org"
fi
-# Push NPM packages to Nexus repository
-for NPM_LIST in "${NXS_NPM_LISTS[@]}"; do
- push_npm "${NPM_LIST}"
-done
-popd
-
###############################
## Populate PyPi repository #
###############################
-
-pushd ${NXS_SRC_PYPI_DIR}
-for PYPI_LIST in "${NXS_PYPI_LISTS[@]}"; do
- push_pip "${PYPI_LIST}"
-done
-popd
+if [ $PYPI_PUSH == "true" ]; then
+ pushd ${NXS_SRC_PYPI_DIR}
+ for PYPI_LIST in "${NXS_PYPI_LISTS[@]}"; do
+ push_pip "${PYPI_LIST}"
+ done
+ popd
+fi
###############################
## Populate Docker repository #
###############################
-# Login to simulated docker registries
-# Push images to private nexus based on the list
-# Images from default registry need to be tagged to private registry
-# and those without defined repository in tag uses default repository 'library'
+# Login to docker registry simulated by Nexus container
+# Push images to private nexus based on the lists
+# All images need to be tagged to simulated registry
+# and those without defined repository in tag use default repository 'library'
+docker_login
for DOCKER_IMG_LIST in "${NXS_DOCKER_IMG_LISTS[@]}"; do
- docker_login "${DOCKER_IMG_LIST}"
push_docker "${DOCKER_IMG_LIST}"
done
# Stop the Nexus and cleanup #
##############################
-echo "Stopping Nexus and returning backups"
+echo "Stopping Nexus"
# Stop the Nexus
docker stop ${NEXUS_CONT_ID} > /dev/null
-# Return backed up configuration files
-mv -f "/etc/${HOSTS_BACKUP}" /etc/hosts
-
-if [ -f ~/.docker/${DOCKER_CONF_BACKUP} ]; then
- mv -f ~/.docker/${DOCKER_CONF_BACKUP} ~/.docker/config.json
-fi
-
-# Return default settings
-npm config set registry "https://registry.npmjs.org"
+# Drop temporary Docker client config dir
+rm -rf ${DOCKER_CONFIG_DIR}
echo "Nexus blob is built"
exit 0
Code Review / oom / offline-installer.git / blobdiff