3 # COPYRIGHT NOTICE STARTS HERE
5 # Copyright 2018-2020© Samsung Electronics Co., Ltd.
7 # Licensed under the Apache License, Version 2.0 (the "License");
8 # you may not use this file except in compliance with the License.
9 # You may obtain a copy of the License at
11 # http://www.apache.org/licenses/LICENSE-2.0
13 # Unless required by applicable law or agreed to in writing, software
14 # distributed under the License is distributed on an "AS IS" BASIS,
15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
19 # COPYRIGHT NOTICE ENDS HERE
21 ### This script prepares Nexus repositories data blobs for ONAP
23 ## The script requires following dependencies are installed: nodejs, jq, docker, twine, expect
24 ## All required resources are expected in the upper directory created during
25 ## download procedure as DATA_DIR or in the directory given as --input-directory
26 ## All lists used must be in project data_lists directory or in the directory given
27 ## as --resource-list-directory
32 TIMESTAMP="date +'%Y-%m-%d_%H-%M-%S'"
33 SCRIPT_LOG="/tmp/$(basename $0)_$(eval ${TIMESTAMP}).log"
36 exec &> >(tee -a "${SCRIPT_LOG}")
38 # Nexus repository properties
40 NEXUS_HOST="127.0.0.1"
41 NEXUS_EXPOSED_PORT="8081"
42 NEXUS_PORT=${NEXUS_EXPOSED_PORT}
43 NEXUS_DOCKER_EXPOSED_PORT="8082"
44 NEXUS_DOCKER_PORT=${NEXUS_DOCKER_EXPOSED_PORT}
45 DEFAULT_REGISTRY="docker.io"
47 # Nexus repository credentials
49 NEXUS_PASSWORD=admin123
50 NEXUS_EMAIL=admin@example.org
53 LOCAL_PATH="$(readlink -f $(dirname ${0}))"
59 DATA_DIR="$(realpath ${LOCAL_PATH}/../../resources)"
60 NEXUS_DATA_DIR="${DATA_DIR}/nexus_data"
61 LISTS_DIR="${LOCAL_PATH}/data_lists"
63 # Required dependencies
68 Usage: $(basename $0) [OPTION...] [FILE]...
70 This script prepares Nexus repositories data blobs for ONAP
72 Following dependencies are required: nodejs, jq, docker, twine, expect
73 By default, without any lists or dirs provided, the resources are expected as downloaded
74 during download process and default lists will be used to build the Nexus blob in the same
78 $(basename $0) --input-directory </path/to/downloaded/files/dir> -ld --output-directory
79 </path/to/output/dir> --resource-list-directory </path/to/dir/with/resource/list>
80 # Docker images, npms and pypi packages will be loaded from specified directory
81 # and the blob is created
82 $(basename $0) -d </path/to/docker/images/list> -d </path/to/another/docker/images/list>
83 -n </path/to/npm/list> -p </path/to/pip/list>
84 # Docker images, npms and pypi packages will be pushed to Nexus based and provided data
85 # lists (multiple lists can be provided)
87 -d | --docker use specific list of docker images to be pushed into Nexus
88 (in case of -ld used, this list will be used for loading of
90 -h | --help print this usage
91 -i | --input-directory use specific directory containing resources needed to
93 The structure of this directory must organized as described
95 -ld | --load-docker-images load docker images from resource directory
96 -n | --npm list of npm packages to be pushed into Nexus
97 -o | --output-directory use specific directory for the target blob
98 -p | --pypi use specific list of pypi packages to be pushed into Nexus
99 -rl | --resource-list-directory use specific directory with docker, pypi and npm lists
100 -c | --container-name use specific Nexus docker container name
101 -NP | --nexus-port use specific port for published Nexus service
102 -DP | --docker-port use specific port for published Nexus docker registry port
107 load_docker_images () {
108 for ARCHIVE in $(sed $'s/\r// ; /^#/d ; s/\:/\_/g ; s/\//\_/g ; s/$/\.tar/g' ${1} | awk '{ print $1 }'); do
109 docker load -i ${NXS_SRC_DOCKER_IMG_DIR}/${ARCHIVE}
114 # Configure NPM registry to our Nexus repository
115 echo "Configure NPM registry to ${NPM_REGISTRY}"
116 npm config set registry "${NPM_REGISTRY}"
118 # Login to NPM registry
119 /usr/bin/expect <<- EOF
122 send "${NEXUS_USERNAME}\n"
124 send "${NEXUS_PASSWORD}\n"
126 send "${NEXUS_EMAIL}\n"
132 # Patch problematic package
133 PATCHED_NPM="$(grep tsscmp ${1} | sed $'s/\r// ; s/\\@/\-/ ; s/$/\.tgz/')"
134 if [[ ! -z "${PATCHED_NPM}" ]] && ! zgrep -aq "${NPM_REGISTRY}" "${PATCHED_NPM}" 2>/dev/null
136 tar xzf "${PATCHED_NPM}"
137 rm -f "${PATCHED_NPM}"
138 sed -i 's|\"registry\":\ \".*\"|\"registry\":\ \"'"${NPM_REGISTRY}"'\"|g' package/package.json
139 tar -zcf "${PATCHED_NPM}" package
145 for ARCHIVE in $(sed $'s/\r// ; s/\\@/\-/g ; s/$/\.tgz/g' ${1}); do
146 npm publish --access public ${ARCHIVE} > /dev/null
147 echo "NPM ${ARCHIVE} pushed to Nexus"
152 for PACKAGE in $(sed $'s/\r//; s/==/-/' ${1}); do
153 twine upload -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" --repository-url ${PYPI_REGISTRY} ${PACKAGE}* > /dev/null
154 echo "PYPI ${PACKAGE} pushed to Nexus"
159 if ! grep -wqs ${DOCKER_REGISTRY} ~/.docker/config.json; then
160 echo "Docker login to ${DOCKER_REGISTRY}"
161 echo -n "${NEXUS_PASSWORD}" | docker login -u "${NEXUS_USERNAME}" --password-stdin ${DOCKER_REGISTRY} > /dev/null
166 for IMAGE in $(sed $'s/\r// ; /^#/d' ${1} | awk '{ print $1 }'); do
168 if [[ ${IMAGE} != *"/"* ]]; then
169 PUSH="${DOCKER_REGISTRY}/library/${IMAGE}"
170 elif [[ ${IMAGE} == *"${DEFAULT_REGISTRY}"* ]]; then
171 if [[ ${IMAGE} == *"/"*"/"* ]]; then
172 PUSH="$(sed 's/'"${DEFAULT_REGISTRY}"'/'"${DOCKER_REGISTRY}"'/' <<< ${IMAGE})"
174 PUSH="$(sed 's/'"${DEFAULT_REGISTRY}"'/'"${DOCKER_REGISTRY}"'\/library/' <<< ${IMAGE})"
176 elif [[ -z $(sed -n '/\.[^/].*\//p' <<< ${IMAGE}) ]]; then
177 PUSH="${DOCKER_REGISTRY}/${IMAGE}"
179 # substitute all host names with $DOCKER_REGISTRY
180 repo_host=$(sed -e 's/\/.*$//' <<< ${IMAGE})
181 PUSH="$(sed -e 's/'"${repo_host}"'/'"${DOCKER_REGISTRY}"'/' <<< ${IMAGE})"
183 docker tag ${IMAGE} ${PUSH}
187 echo "${IMAGE} pushed as ${PUSH} to Nexus"
191 validate_container_name () {
192 # Verify $1 is a valid hostname
193 if ! echo "${1}" | egrep -q "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$";
195 echo "ERROR: ${1} is not a valid name!"
200 while [ "${1}" != "" ]; do
202 -d | --docker ) shift
203 NXS_DOCKER_IMG_LISTS+=("$(realpath ${1})")
205 -i | --input-directory ) shift
206 DATA_DIR="$(realpath ${1})"
208 -ld | --load-docker-images ) DOCKER_LOAD="true"
210 -n | --npm ) NPM_PUSH="true"
211 COMMANDS+=(expect npm)
213 NXS_NPM_LISTS+=("$(realpath ${1})")
215 -c | --container-name ) shift
216 validate_container_name "${1}"
219 -o | --output-directory ) shift
220 NEXUS_DATA_DIR="$(realpath ${1})"
222 -p | --pypi ) PYPI_PUSH="true"
225 NXS_PYPI_LISTS+=("$(realpath ${1})")
227 -rl | --resource-list-directory ) shift
228 LISTS_DIR="$(realpath ${1})"
230 -NP | --nexus-port ) shift
233 -DP | --docker-port ) shift
234 NEXUS_DOCKER_PORT="${1}"
244 # Verify all dependencies are available in PATH
246 for cmd in ${COMMANDS[*]}; do
247 command -v $cmd >/dev/null 2>&1 || FAILED_COMMANDS+=($cmd)
250 if [ ${#FAILED_COMMANDS[*]} -gt 0 ]; then
251 echo "Following commands where not found in PATH and are required:"
252 echo ${FAILED_COMMANDS[*]}
257 # Nexus repository locations
258 NPM_REGISTRY="http://${NEXUS_HOST}:${NEXUS_PORT}/repository/npm-private/"
259 PYPI_REGISTRY="http://${NEXUS_HOST}:${NEXUS_PORT}/repository/pypi-private/"
260 DOCKER_REGISTRY="${NEXUS_HOST}:${NEXUS_DOCKER_PORT}"
262 # Setup directories with resources for docker, npm and pypi
263 NXS_SRC_DOCKER_IMG_DIR="${DATA_DIR}/offline_data/docker_images_for_nexus"
264 NXS_SRC_NPM_DIR="${DATA_DIR}/offline_data/npm_tar"
265 NXS_SRC_PYPI_DIR="${DATA_DIR}/offline_data/pypi"
267 # Setup specific resources lists
268 NXS_INFRA_LIST="${LISTS_DIR}/infra_docker_images.list"
269 NXS_DOCKER_IMG_LIST="${LISTS_DIR}/onap_docker_images.list"
270 NXS_RKE_DOCKER_IMG_LIST="${LISTS_DIR}/rke_docker_images.list"
271 NXS_K8S_DOCKER_IMG_LIST="${LISTS_DIR}/k8s_docker_images.list"
273 # Setup Nexus image used for build and install infra
274 NEXUS_IMAGE="$(grep sonatype/nexus3 ${NXS_INFRA_LIST})"
275 NEXUS_IMAGE_TAR="${DATA_DIR}/offline_data/docker_images_infra/$(sed 's/\//\_/ ; s/$/\.tar/ ; s/\:/\_/' <<< ${NEXUS_IMAGE})"
277 # Set default lists if nothing specific defined by user
278 if [ ${#NXS_DOCKER_IMG_LISTS[@]} -eq 0 ]; then
279 NXS_DOCKER_IMG_LISTS=("${NXS_DOCKER_IMG_LIST}" "${NXS_RKE_DOCKER_IMG_LIST}" "${NXS_K8S_DOCKER_IMG_LIST}")
282 # Backup the current docker registry settings
283 if [ -f ~/.docker/config.json ]; then
284 DOCKER_CONF_BACKUP="$(eval ${TIMESTAMP}_config.json.bk)"
285 mv ~/.docker/config.json ~/.docker/${DOCKER_CONF_BACKUP}
288 # Setup default ports published to host as docker registry
289 PUBLISHED_PORTS="-p ${NEXUS_PORT}:${NEXUS_EXPOSED_PORT} -p ${NEXUS_DOCKER_PORT}:${NEXUS_DOCKER_EXPOSED_PORT}"
291 # Nexus repository configuration setup
292 NEXUS_CONFIG_GROOVY='import org.sonatype.nexus.security.realm.RealmManager
293 import org.sonatype.nexus.repository.attributes.AttributesFacet
294 import org.sonatype.nexus.security.user.UserManager
295 import org.sonatype.nexus.repository.manager.RepositoryManager
296 import org.sonatype.nexus.security.user.UserNotFoundException
297 /* Use the container to look up some services. */
298 realmManager = container.lookup(RealmManager.class)
299 userManager = container.lookup(UserManager.class, "default") //default user manager
300 repositoryManager = container.lookup(RepositoryManager.class)
301 /* Managers are used when scripting api cannot. Note that scripting api can only create mostly, and that creation methods return objects of created entities. */
302 /* Perform cleanup by removing all repos and users. Realms do not need to be re-disabled, admin and anonymous user will not be removed. */
303 userManager.listUserIds().each({ id ->
304 if (id != "anonymous" && id != "admin")
305 userManager.deleteUser(id)
307 repositoryManager.browse().each {
308 repositoryManager.delete(it.getName())
310 /* Add bearer token realms at the end of realm lists... */
311 realmManager.enableRealm("NpmToken")
312 realmManager.enableRealm("DockerToken")
313 realmManager.enableRealm("PypiToken")
314 /* Create the docker user. */
315 security.addUser("docker", "docker", "docker", "docker@example.com", true, "docker", ["nx-anonymous"])
316 /* Create docker, npm and pypi repositories. Their default configuration should be compliant with our requirements, except the docker registry creation. */
317 repository.createNpmHosted("npm-private")
318 repository.createPyPiHosted("pypi-private")
319 def r = repository.createDockerHosted("onap", 8082, 0)
320 /* force basic authentication true by default, must set to false for docker repo. */
321 conf=r.getConfiguration()
322 conf.attributes("docker").set("forceBasicAuth", false)
323 repositoryManager.update(conf)'
325 # Prepare the Nexus configuration
326 NEXUS_CONFIG=$(echo "${NEXUS_CONFIG_GROOVY}" | jq -Rsc '{"name":"configure", "type":"groovy", "content":.}')
328 #################################
329 # Docker repository preparation #
330 #################################
332 if [ "${DOCKER_LOAD}" == "true" ]; then
333 # Load predefined Nexus image
334 docker load -i ${NEXUS_IMAGE_TAR}
335 # Load all necessary images
336 for DOCKER_IMG_LIST in "${NXS_DOCKER_IMG_LISTS[@]}"; do
337 load_docker_images "${DOCKER_IMG_LIST}"
341 ################################
342 # Nexus repository preparation #
343 ################################
345 # Prepare nexus-data directory
346 if [ -d ${NEXUS_DATA_DIR} ]; then
347 if [ "$(docker ps -q -f name="${NEXUS_DOMAIN}")" ]; then
348 echo "Removing container ${NEXUS_DOMAIN}"
349 docker rm -f $(docker ps -aq -f name="${NEXUS_DOMAIN}")
351 pushd ${NEXUS_DATA_DIR}/..
352 NXS_BACKUP="$(eval ${TIMESTAMP})_$(basename ${NEXUS_DATA_DIR})_bk"
353 mv ${NEXUS_DATA_DIR} "${NXS_BACKUP}"
354 echo "${NEXUS_DATA_DIR} already exists - backing up to ${NXS_BACKUP}"
358 mkdir -p ${NEXUS_DATA_DIR}
359 chown 200:200 ${NEXUS_DATA_DIR}
360 chmod 777 ${NEXUS_DATA_DIR}
362 # Save Nexus version to prevent/catch data incompatibility
363 # Adding commit informations to have link to data from which the blob was built
364 cat >> ${NEXUS_DATA_DIR}/nexus.ver << INFO
365 nexus_image=$(docker image ls ${NEXUS_IMAGE} --no-trunc --format "{{.Repository}}:{{.Tag}}\nnexus_image_digest={{.ID}}")
366 $(for INDEX in ${!NXS_DOCKER_IMG_LISTS[@]}; do printf 'used_image_list%s=%s\n' "$INDEX" "$(sed 's/^.*\/\(.*\)$/\1/' <<< ${NXS_DOCKER_IMG_LISTS[$INDEX]})"; done)
367 $(sed -n 's/^.*OOM\ commit\ /oom_repo_commit=/p' ${NXS_DOCKER_IMG_LISTS[@]})
368 installer_repo_commit=$(git --git-dir="${LOCAL_PATH}/../.git" rev-parse HEAD)
372 NEXUS_CONT_ID=$(docker run -d --rm -v ${NEXUS_DATA_DIR}:/nexus-data:rw --name ${NEXUS_DOMAIN} ${PUBLISHED_PORTS} ${NEXUS_IMAGE})
373 echo "Waiting for Nexus to fully start"
374 until curl -su ${NEXUS_USERNAME}:${NEXUS_PASSWORD} http://${NEXUS_HOST}:${NEXUS_PORT}/service/metrics/healthcheck | grep '"healthy":true' > /dev/null ; do
378 echo -e "\nNexus started"
380 # Configure the nexus repository
381 curl -sX POST --header 'Content-Type: application/json' --data-binary "${NEXUS_CONFIG}" http://${NEXUS_USERNAME}:${NEXUS_PASSWORD}@${NEXUS_HOST}:${NEXUS_PORT}/service/rest/v1/script
382 curl -sX POST --header "Content-Type: text/plain" http://${NEXUS_USERNAME}:${NEXUS_PASSWORD}@${NEXUS_HOST}:${NEXUS_PORT}/service/rest/v1/script/configure/run > /dev/null
384 ###########################
385 # Populate NPM repository #
386 ###########################
387 if [ $NPM_PUSH == "true" ]; then
389 pushd ${NXS_SRC_NPM_DIR}
390 for NPM_LIST in "${NXS_NPM_LISTS[@]}"; do
391 patch_npm "${NPM_LIST}"
392 push_npm "${NPM_LIST}"
395 # Return default settings
397 npm config set registry "https://registry.npmjs.org"
400 ###############################
401 ## Populate PyPi repository #
402 ###############################
403 if [ $PYPI_PUSH == "true" ]; then
404 pushd ${NXS_SRC_PYPI_DIR}
405 for PYPI_LIST in "${NXS_PYPI_LISTS[@]}"; do
406 push_pip "${PYPI_LIST}"
411 ###############################
412 ## Populate Docker repository #
413 ###############################
415 # Login to docker registry simulated by Nexus container
416 # Push images to private nexus based on the lists
417 # All images need to be tagged to simulated registry
418 # and those without defined repository in tag use default repository 'library'
420 for DOCKER_IMG_LIST in "${NXS_DOCKER_IMG_LISTS[@]}"; do
421 push_docker "${DOCKER_IMG_LIST}"
424 ##############################
425 # Stop the Nexus and cleanup #
426 ##############################
428 echo "Stopping Nexus and returning backups"
431 docker stop ${NEXUS_CONT_ID} > /dev/null
433 if [ -f ~/.docker/${DOCKER_CONF_BACKUP} ]; then
434 mv -f ~/.docker/${DOCKER_CONF_BACKUP} ~/.docker/config.json
437 echo "Nexus blob is built"