3 # COPYRIGHT NOTICE STARTS HERE
5 # Copyright 2018-2019 © Samsung Electronics Co., Ltd.
7 # Licensed under the Apache License, Version 2.0 (the "License");
8 # you may not use this file except in compliance with the License.
9 # You may obtain a copy of the License at
11 # http://www.apache.org/licenses/LICENSE-2.0
13 # Unless required by applicable law or agreed to in writing, software
14 # distributed under the License is distributed on an "AS IS" BASIS,
15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
19 # COPYRIGHT NOTICE ENDS HERE
21 ### This script prepares Nexus repositories data blobs for ONAP
23 ## The script requires following dependencies are installed: nodejs, jq, docker, twine, expect
24 ## All required resources are expected in the upper directory created during
25 ## download procedure as DATA_DIR or in the directory given as --input-directory
26 ## All lists used must be in project data_lists directory or in the directory given
27 ## as --resource-list-directory
32 TIMESTAMP="date +'%Y-%m-%d_%H-%M-%S'"
33 SCRIPT_LOG="/tmp/$(basename $0)_$(eval ${TIMESTAMP}).log"
36 exec &> >(tee -a "${SCRIPT_LOG}")
38 # Nexus repository location
41 NEXUS_DOCKER_PORT="8082"
42 NPM_REGISTRY="http://${NEXUS_DOMAIN}:${NEXUS_PORT}/repository/npm-private/"
43 PYPI_REGISTRY="http://${NEXUS_DOMAIN}:${NEXUS_PORT}/repository/pypi-private/"
44 DOCKER_REGISTRY="${NEXUS_DOMAIN}:${NEXUS_DOCKER_PORT}"
45 DEFAULT_REGISTRY="docker.io"
47 # Nexus repository credentials
49 NEXUS_PASSWORD=admin123
50 NEXUS_EMAIL=admin@example.org
53 LOCAL_PATH="$(readlink -f $(dirname ${0}))"
59 DATA_DIR="$(realpath ${LOCAL_PATH}/../../resources)"
60 NEXUS_DATA_DIR="${DATA_DIR}/nexus_data"
61 LISTS_DIR="${LOCAL_PATH}/data_lists"
63 # Required dependencies
68 Usage: $(basename $0) [OPTION...] [FILE]...
70 This script prepares Nexus repositories data blobs for ONAP
72 Following dependencies are required: nodejs, jq, docker, twine, expect
73 By default, without any lists or dirs provided, the resources are expected as downloaded
74 during download process and default lists will be used to build the Nexus blob in the same
78 $(basename $0) --input-directory </path/to/downloaded/files/dir> -ld --output-directory
79 </path/to/output/dir> --resource-list-directory </path/to/dir/with/resource/list>
80 # Docker images, npms and pypi packages will be loaded from specified directory
81 # and the blob is created
82 $(basename $0) -d </path/to/docker/images/list> -d </path/to/another/docker/images/list>
83 -n </path/to/npm/list> -p </path/to/pip/list>
84 # Docker images, npms and pypi packages will be pushed to Nexus based and provided data
85 # lists (multiple lists can be provided)
87 -d | --docker use specific list of docker images to be pushed into Nexus
88 (in case of -ld used, this list will be used for loading of
90 -h | --help print this usage
91 -i | --input-directory use specific directory containing resources needed to
93 The structure of this directory must organized as described
95 -ld | --load-docker-images load docker images from resource directory
96 -n | --npm list of npm packages to be pushed into Nexus
97 -o | --output-directory use specific directory for the target blob
98 -p | --pypi use specific list of pypi packages to be pushed into Nexus
99 -rl | --resource-list-directory use specific directory with docker, pypi and npm lists
105 for REGISTRY in $(sed -n '/\.[^/].*\//p' ${1} | sed -e 's/\/.*$//' | sort -u | grep -v ${DEFAULT_REGISTRY} || true) ${NEXUS_PORT}; do
106 if [[ ${REGISTRY} != *":"* ]]; then
107 if [[ ${PUBLISHED_PORTS} != *"80:${NEXUS_DOCKER_PORT}"* ]]; then
108 PUBLISHED_PORTS="${PUBLISHED_PORTS} -p 80:${NEXUS_DOCKER_PORT}"
111 REGISTRY_PORT="$(sed 's/^.*\:\([[:digit:]]*\)$/\1/' <<< ${REGISTRY})"
112 if [[ ${PUBLISHED_PORTS} != *"${REGISTRY_PORT}:${NEXUS_DOCKER_PORT}"* ]]; then
113 PUBLISHED_PORTS="${PUBLISHED_PORTS} -p ${REGISTRY_PORT}:${NEXUS_DOCKER_PORT}"
120 SIMUL_HOSTS=($(sed -n '/\.[^/].*\//p' ${1} | sed -e 's/\/.*$// ; s/:.*$//' | sort -u | grep -v ${DEFAULT_REGISTRY} || true ) ${NEXUS_DOMAIN})
121 for HOST in "${SIMUL_HOSTS[@]}"; do
122 if ! grep -wq ${HOST} /etc/hosts; then
123 echo "127.0.0.1 ${HOST}" >> /etc/hosts
128 load_docker_images () {
129 for ARCHIVE in $(sed $'s/\r// ; /^#/d ; s/\:/\_/g ; s/\//\_/g ; s/$/\.tar/g' ${1} | awk '{ print $1 }'); do
130 docker load -i ${NXS_SRC_DOCKER_IMG_DIR}/${ARCHIVE}
135 # Configure NPM registry to our Nexus repository
136 echo "Configure NPM registry to ${NPM_REGISTRY}"
137 npm config set registry "${NPM_REGISTRY}"
139 # Login to NPM registry
140 /usr/bin/expect <<- EOF
143 send "${NEXUS_USERNAME}\n"
145 send "${NEXUS_PASSWORD}\n"
147 send "${NEXUS_EMAIL}\n"
153 # Patch problematic package
154 PATCHED_NPM="$(grep tsscmp ${1} | sed $'s/\r// ; s/\\@/\-/ ; s/$/\.tgz/')"
155 if [[ ! -z "${PATCHED_NPM}" ]] && ! zgrep -aq "${NPM_REGISTRY}" "${PATCHED_NPM}" 2>/dev/null
157 tar xzf "${PATCHED_NPM}"
158 rm -f "${PATCHED_NPM}"
159 sed -i 's|\"registry\":\ \".*\"|\"registry\":\ \"'"${NPM_REGISTRY}"'\"|g' package/package.json
160 tar -zcf "${PATCHED_NPM}" package
166 for ARCHIVE in $(sed $'s/\r// ; s/\\@/\-/g ; s/$/\.tgz/g' ${1}); do
167 npm publish --access public ${ARCHIVE} > /dev/null
168 echo "NPM ${ARCHIVE} pushed to Nexus"
173 for PACKAGE in $(sed $'s/\r//; s/==/-/' ${1}); do
174 twine upload -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" --repository-url ${PYPI_REGISTRY} ${PACKAGE}* > /dev/null
175 echo "PYPI ${PACKAGE} pushed to Nexus"
180 for REGISTRY in $(sed -n '/\.[^/].*\//p' ${1} | sed -e 's/\/.*$//' | sort -u | grep -v ${DEFAULT_REGISTRY}) ${DOCKER_REGISTRY}; do
181 if ! grep -wqs ${REGISTRY} ~/.docker/config.json; then
182 echo "Docker login to ${REGISTRY}"
183 echo -n "${NEXUS_PASSWORD}" | docker login -u "${NEXUS_USERNAME}" --password-stdin ${REGISTRY} > /dev/null
189 for IMAGE in $(sed $'s/\r// ; /^#/d' ${1} | awk '{ print $1 }'); do
191 if [[ ${IMAGE} != *"/"* ]]; then
192 PUSH="${DOCKER_REGISTRY}/library/${IMAGE}"
193 elif [[ ${IMAGE} == *"${DEFAULT_REGISTRY}"* ]]; then
194 if [[ ${IMAGE} == *"/"*"/"* ]]; then
195 PUSH="$(sed 's/'"${DEFAULT_REGISTRY}"'/'"${DOCKER_REGISTRY}"'/' <<< ${IMAGE})"
197 PUSH="$(sed 's/'"${DEFAULT_REGISTRY}"'/'"${DOCKER_REGISTRY}"'\/library/' <<< ${IMAGE})"
199 elif [[ -z $(sed -n '/\.[^/].*\//p' <<< ${IMAGE}) ]]; then
200 PUSH="${DOCKER_REGISTRY}/${IMAGE}"
202 if [[ ! -z ${PUSH} ]]; then
203 docker tag ${IMAGE} ${PUSH}
208 echo "${IMAGE} pushed as ${PUSH} to Nexus"
212 while [ "${1}" != "" ]; do
214 -d | --docker ) shift
215 NXS_DOCKER_IMG_LISTS+=("$(realpath ${1})")
217 -i | --input-directory ) shift
218 DATA_DIR="$(realpath ${1})"
220 -ld | --load-docker-images ) DOCKER_LOAD="true"
222 -n | --npm ) NPM_PUSH="true"
223 COMMANDS+=(expect npm)
225 NXS_NPM_LISTS+=("$(realpath ${1})")
227 -o | --output-directory ) shift
228 NEXUS_DATA_DIR="$(realpath ${1})"
230 -p | --pypi ) PYPI_PUSH="true"
233 NXS_PYPI_LISTS+=("$(realpath ${1})")
235 -rl | --resource-list-directory ) shift
236 LISTS_DIR="$(realpath ${1})"
246 # Verify all dependencies are available in PATH
248 for cmd in ${COMMANDS[*]}; do
249 command -v $cmd >/dev/null 2>&1 || FAILED_COMMANDS+=($cmd)
252 if [ ${#FAILED_COMMANDS[*]} -gt 0 ]; then
253 echo "Following commands where not found in PATH and are required:"
254 echo ${FAILED_COMMANDS[*]}
259 # Setup directories with resources for docker, npm and pypi
260 NXS_SRC_DOCKER_IMG_DIR="${DATA_DIR}/offline_data/docker_images_for_nexus"
261 NXS_SRC_NPM_DIR="${DATA_DIR}/offline_data/npm_tar"
262 NXS_SRC_PYPI_DIR="${DATA_DIR}/offline_data/pypi"
264 # Setup specific resources lists
265 NXS_INFRA_LIST="${LISTS_DIR}/infra_docker_images.list"
266 NXS_DOCKER_IMG_LIST="${LISTS_DIR}/onap_docker_images.list"
267 NXS_RKE_DOCKER_IMG_LIST="${LISTS_DIR}/rke_docker_images.list"
268 NXS_K8S_DOCKER_IMG_LIST="${LISTS_DIR}/k8s_docker_images.list"
270 # Setup Nexus image used for build and install infra
271 NEXUS_IMAGE="$(grep sonatype/nexus3 ${NXS_INFRA_LIST})"
272 NEXUS_IMAGE_TAR="${DATA_DIR}/offline_data/docker_images_infra/$(sed 's/\//\_/ ; s/$/\.tar/ ; s/\:/\_/' <<< ${NEXUS_IMAGE})"
274 # Set default lists if nothing specific defined by user
275 if [ ${#NXS_DOCKER_IMG_LISTS[@]} -eq 0 ]; then
276 NXS_DOCKER_IMG_LISTS=("${NXS_DOCKER_IMG_LIST}" "${NXS_RKE_DOCKER_IMG_LIST}" "${NXS_K8S_DOCKER_IMG_LIST}")
280 HOSTS_BACKUP="$(eval ${TIMESTAMP}_hosts.bk)"
281 cp /etc/hosts /etc/${HOSTS_BACKUP}
283 # Backup the current docker registry settings
284 if [ -f ~/.docker/config.json ]; then
285 DOCKER_CONF_BACKUP="$(eval ${TIMESTAMP}_config.json.bk)"
286 mv ~/.docker/config.json ~/.docker/${DOCKER_CONF_BACKUP}
289 # Setup default ports published to host as docker registry
290 PUBLISHED_PORTS="-p ${NEXUS_PORT}:${NEXUS_PORT} -p ${NEXUS_DOCKER_PORT}:${NEXUS_DOCKER_PORT}"
292 # Setup additional ports published to host based on simulated docker registries
293 # Setup simulated domain names to be able to push all to private Nexus repository
294 for DOCKER_IMG_LIST in "${NXS_DOCKER_IMG_LISTS[@]}"; do
295 publish_ports "${DOCKER_IMG_LIST}"
296 simulated_hosts "${DOCKER_IMG_LIST}"
299 # Nexus repository configuration setup
300 NEXUS_CONFIG_GROOVY='import org.sonatype.nexus.security.realm.RealmManager
301 import org.sonatype.nexus.repository.attributes.AttributesFacet
302 import org.sonatype.nexus.security.user.UserManager
303 import org.sonatype.nexus.repository.manager.RepositoryManager
304 import org.sonatype.nexus.security.user.UserNotFoundException
305 /* Use the container to look up some services. */
306 realmManager = container.lookup(RealmManager.class)
307 userManager = container.lookup(UserManager.class, "default") //default user manager
308 repositoryManager = container.lookup(RepositoryManager.class)
309 /* Managers are used when scripting api cannot. Note that scripting api can only create mostly, and that creation methods return objects of created entities. */
310 /* Perform cleanup by removing all repos and users. Realms do not need to be re-disabled, admin and anonymous user will not be removed. */
311 userManager.listUserIds().each({ id ->
312 if (id != "anonymous" && id != "admin")
313 userManager.deleteUser(id)
315 repositoryManager.browse().each {
316 repositoryManager.delete(it.getName())
318 /* Add bearer token realms at the end of realm lists... */
319 realmManager.enableRealm("NpmToken")
320 realmManager.enableRealm("DockerToken")
321 realmManager.enableRealm("PypiToken")
322 /* Create the docker user. */
323 security.addUser("docker", "docker", "docker", "docker@example.com", true, "docker", ["nx-anonymous"])
324 /* Create docker, npm and pypi repositories. Their default configuration should be compliant with our requirements, except the docker registry creation. */
325 repository.createNpmHosted("npm-private")
326 repository.createPyPiHosted("pypi-private")
327 def r = repository.createDockerHosted("onap", 8082, 0)
328 /* force basic authentication true by default, must set to false for docker repo. */
329 conf=r.getConfiguration()
330 conf.attributes("docker").set("forceBasicAuth", false)
331 repositoryManager.update(conf)'
333 # Prepare the Nexus configuration
334 NEXUS_CONFIG=$(echo "${NEXUS_CONFIG_GROOVY}" | jq -Rsc '{"name":"configure", "type":"groovy", "content":.}')
336 #################################
337 # Docker repository preparation #
338 #################################
340 if [ "${DOCKER_LOAD}" == "true" ]; then
341 # Load predefined Nexus image
342 docker load -i ${NEXUS_IMAGE_TAR}
343 # Load all necessary images
344 for DOCKER_IMG_LIST in "${NXS_DOCKER_IMG_LISTS[@]}"; do
345 load_docker_images "${DOCKER_IMG_LIST}"
349 ################################
350 # Nexus repository preparation #
351 ################################
353 # Prepare nexus-data directory
354 if [ -d ${NEXUS_DATA_DIR} ]; then
355 if [ "$(docker ps -q -f name="${NEXUS_DOMAIN}")" ]; then
356 echo "Removing container ${NEXUS_DOMAIN}"
357 docker rm -f $(docker ps -aq -f name="${NEXUS_DOMAIN}")
359 pushd ${NEXUS_DATA_DIR}/..
360 NXS_BACKUP="$(eval ${TIMESTAMP})_$(basename ${NEXUS_DATA_DIR})_bk"
361 mv ${NEXUS_DATA_DIR} "${NXS_BACKUP}"
362 echo "${NEXUS_DATA_DIR} already exists - backing up to ${NXS_BACKUP}"
366 mkdir -p ${NEXUS_DATA_DIR}
367 chown 200:200 ${NEXUS_DATA_DIR}
368 chmod 777 ${NEXUS_DATA_DIR}
370 # Save Nexus version to prevent/catch data incompatibility
371 # Adding commit informations to have link to data from which the blob was built
372 cat >> ${NEXUS_DATA_DIR}/nexus.ver << INFO
373 nexus_image=$(docker image ls ${NEXUS_IMAGE} --no-trunc --format "{{.Repository}}:{{.Tag}}\nnexus_image_digest={{.ID}}")
374 $(for INDEX in ${!NXS_DOCKER_IMG_LISTS[@]}; do printf 'used_image_list%s=%s\n' "$INDEX" "$(sed 's/^.*\/\(.*\)$/\1/' <<< ${NXS_DOCKER_IMG_LISTS[$INDEX]})"; done)
375 $(sed -n 's/^.*OOM\ commit\ /oom_repo_commit=/p' ${NXS_DOCKER_IMG_LISTS[@]})
376 installer_repo_commit=$(git --git-dir="${LOCAL_PATH}/../.git" rev-parse HEAD)
380 NEXUS_CONT_ID=$(docker run -d --rm -v ${NEXUS_DATA_DIR}:/nexus-data:rw --name ${NEXUS_DOMAIN} ${PUBLISHED_PORTS} ${NEXUS_IMAGE})
381 echo "Waiting for Nexus to fully start"
382 until curl -su ${NEXUS_USERNAME}:${NEXUS_PASSWORD} http://${NEXUS_DOMAIN}:${NEXUS_PORT}/service/metrics/healthcheck | grep '"healthy":true' > /dev/null ; do
386 echo -e "\nNexus started"
388 # Configure the nexus repository
389 curl -sX POST --header 'Content-Type: application/json' --data-binary "${NEXUS_CONFIG}" http://${NEXUS_USERNAME}:${NEXUS_PASSWORD}@${NEXUS_DOMAIN}:${NEXUS_PORT}/service/rest/v1/script
390 curl -sX POST --header "Content-Type: text/plain" http://${NEXUS_USERNAME}:${NEXUS_PASSWORD}@${NEXUS_DOMAIN}:${NEXUS_PORT}/service/rest/v1/script/configure/run > /dev/null
392 ###########################
393 # Populate NPM repository #
394 ###########################
395 if [ $NPM_PUSH == "true" ]; then
397 pushd ${NXS_SRC_NPM_DIR}
398 for NPM_LIST in "${NXS_NPM_LISTS[@]}"; do
399 patch_npm "${NPM_LIST}"
400 push_npm "${NPM_LIST}"
403 # Return default settings
405 npm config set registry "https://registry.npmjs.org"
408 ###############################
409 ## Populate PyPi repository #
410 ###############################
411 if [ $PYPI_PUSH == "true" ]; then
412 pushd ${NXS_SRC_PYPI_DIR}
413 for PYPI_LIST in "${NXS_PYPI_LISTS[@]}"; do
414 push_pip "${PYPI_LIST}"
419 ###############################
420 ## Populate Docker repository #
421 ###############################
423 # Login to simulated docker registries
424 # Push images to private nexus based on the lists
425 # Images from default registry need to be tagged to private registry
426 # and those without defined repository in tag uses default repository 'library'
427 for DOCKER_IMG_LIST in "${NXS_DOCKER_IMG_LISTS[@]}"; do
428 docker_login "${DOCKER_IMG_LIST}"
429 push_docker "${DOCKER_IMG_LIST}"
432 ##############################
433 # Stop the Nexus and cleanup #
434 ##############################
436 echo "Stopping Nexus and returning backups"
439 docker stop ${NEXUS_CONT_ID} > /dev/null
441 # Return backed up configuration files
442 mv -f "/etc/${HOSTS_BACKUP}" /etc/hosts
444 if [ -f ~/.docker/${DOCKER_CONF_BACKUP} ]; then
445 mv -f ~/.docker/${DOCKER_CONF_BACKUP} ~/.docker/config.json
448 echo "Nexus blob is built"