3 # COPYRIGHT NOTICE STARTS HERE
5 # Copyright 2018-2019 © Samsung Electronics Co., Ltd.
7 # Licensed under the Apache License, Version 2.0 (the "License");
8 # you may not use this file except in compliance with the License.
9 # You may obtain a copy of the License at
11 # http://www.apache.org/licenses/LICENSE-2.0
13 # Unless required by applicable law or agreed to in writing, software
14 # distributed under the License is distributed on an "AS IS" BASIS,
15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
19 # COPYRIGHT NOTICE ENDS HERE
21 ### This script prepares Nexus repositories data blobs for ONAP
23 ## The script requires following dependencies are installed: nodejs, jq, docker, twine, expect
24 ## All required resources are expected in the upper directory created during
25 ## download procedure as DATA_DIR or in the directory given as --input-directory
26 ## All lists used must be in project data_lists directory or in the directory given
27 ## as --resource-list-directory
32 TIMESTAMP="date +'%Y-%m-%d_%H-%M-%S'"
33 SCRIPT_LOG="/tmp/$(basename $0)_$(eval ${TIMESTAMP}).log"
36 exec &> >(tee -a "${SCRIPT_LOG}")
38 # Nexus repository location
41 NEXUS_DOCKER_PORT="8082"
42 NPM_REGISTRY="http://${NEXUS_DOMAIN}:${NEXUS_PORT}/repository/npm-private/"
43 PYPI_REGISTRY="http://${NEXUS_DOMAIN}:${NEXUS_PORT}/repository/pypi-private/"
44 DOCKER_REGISTRY="${NEXUS_DOMAIN}:${NEXUS_DOCKER_PORT}"
45 DEFAULT_REGISTRY="docker.io"
47 # Nexus repository credentials
49 NEXUS_PASSWORD=admin123
50 NEXUS_EMAIL=admin@example.org
53 LOCAL_PATH="$(readlink -f $(dirname ${0}))"
57 DATA_DIR="$(realpath ${LOCAL_PATH}/../../resources)"
58 NEXUS_DATA_DIR="${DATA_DIR}/nexus_data"
59 LISTS_DIR="${LOCAL_PATH}/data_lists"
61 # Required dependencies
62 COMMANDS=(jq docker expect npm twine)
66 Usage: $(basename $0) [OPTION...] [FILE]...
68 This script prepares Nexus repositories data blobs for ONAP
70 Following dependencies are required: nodejs, jq, docker, twine, expect
71 By default, without any lists or dirs provided, the resources are expected as downloaded
72 during download process and default lists will be used to build the Nexus blob in the same
76 $(basename $0) --input-directory </path/to/downloaded/files/dir> -ld --output-directory
77 </path/to/output/dir> --resource-list-directory </path/to/dir/with/resource/list>
78 # Docker images, npms and pypi packages will be loaded from specified directory
79 # and the blob is created
80 $(basename $0) -d </path/to/docker/images/list> -d </path/to/another/docker/images/list>
81 -n </path/to/npm/list> -p </path/to/pip/list>
82 # Docker images, npms and pypi packages will be pushed to Nexus based and provided data
83 # lists (multiple lists can be provided)
85 -d | --docker use specific list of docker images to be pushed into Nexus
86 (in case of -ld used, this list will be used for loading of
88 -h | --help print this usage
89 -i | --input-directory use specific directory containing resources needed to
91 The structure of this directory must organized as described
93 -ld | --load-docker-images load docker images from resource directory
94 -n | --npm list of npm packages to be pushed into Nexus
95 -o | --output-directory use specific directory for the target blob
96 -p | --pypi use specific list of pypi packages to be pushed into Nexus
97 -rl | --resource-list-directory use specific directory with docker, pypi and npm lists
103 for REGISTRY in $(sed -n '/\.[^/].*\//p' ${1} | sed -e 's/\/.*$//' | sort -u | grep -v ${DEFAULT_REGISTRY} || true) ${NEXUS_PORT}; do
104 if [[ ${REGISTRY} != *":"* ]]; then
105 if [[ ${PUBLISHED_PORTS} != *"80:${NEXUS_DOCKER_PORT}"* ]]; then
106 PUBLISHED_PORTS="${PUBLISHED_PORTS} -p 80:${NEXUS_DOCKER_PORT}"
109 REGISTRY_PORT="$(sed 's/^.*\:\([[:digit:]]*\)$/\1/' <<< ${REGISTRY})"
110 if [[ ${PUBLISHED_PORTS} != *"${REGISTRY_PORT}:${NEXUS_DOCKER_PORT}"* ]]; then
111 PUBLISHED_PORTS="${PUBLISHED_PORTS} -p ${REGISTRY_PORT}:${NEXUS_DOCKER_PORT}"
118 SIMUL_HOSTS=($(sed -n '/\.[^/].*\//p' ${1} | sed -e 's/\/.*$// ; s/:.*$//' | sort -u | grep -v ${DEFAULT_REGISTRY} || true ) ${NEXUS_DOMAIN})
119 for HOST in "${SIMUL_HOSTS[@]}"; do
120 if ! grep -wq ${HOST} /etc/hosts; then
121 echo "127.0.0.1 ${HOST}" >> /etc/hosts
126 load_docker_images () {
127 for ARCHIVE in $(sed $'s/\r// ; /^#/d ; s/\:/\_/g ; s/\//\_/g ; s/$/\.tar/g' ${1} | awk '{ print $1 }'); do
128 docker load -i ${NXS_SRC_DOCKER_IMG_DIR}/${ARCHIVE}
133 for ARCHIVE in $(sed $'s/\r// ; s/\\@/\-/g ; s/$/\.tgz/g' ${1}); do
134 npm publish --access public ${ARCHIVE} > /dev/null
135 echo "NPM ${ARCHIVE} pushed to Nexus"
140 for PACKAGE in $(sed $'s/\r//; s/==/-/' ${NXS_PYPI_LIST}); do
141 twine upload -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" --repository-url ${PYPI_REGISTRY} ${PACKAGE}*
142 echo "PYPI ${PACKAGE} pushed to Nexus"
147 for REGISTRY in $(sed -n '/\.[^/].*\//p' ${1} | sed -e 's/\/.*$//' | sort -u | grep -v ${DEFAULT_REGISTRY}) ${DOCKER_REGISTRY}; do
148 if ! grep -wqs ${REGISTRY} ~/.docker/config.json; then
149 echo "Docker login to ${REGISTRY}"
150 docker login -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" ${REGISTRY} > /dev/null
156 for IMAGE in $(sed $'s/\r// ; /^#/d' ${1} | awk '{ print $1 }'); do
158 if [[ ${IMAGE} != *"/"* ]]; then
159 PUSH="${DOCKER_REGISTRY}/library/${IMAGE}"
160 elif [[ ${IMAGE} == *"${DEFAULT_REGISTRY}"* ]]; then
161 if [[ ${IMAGE} == *"/"*"/"* ]]; then
162 PUSH="$(sed 's/'"${DEFAULT_REGISTRY}"'/'"${DOCKER_REGISTRY}"'/' <<< ${IMAGE})"
164 PUSH="$(sed 's/'"${DEFAULT_REGISTRY}"'/'"${DOCKER_REGISTRY}"'\/library/' <<< ${IMAGE})"
166 elif [[ -z $(sed -n '/\.[^/].*\//p' <<< ${IMAGE}) ]]; then
167 PUSH="${DOCKER_REGISTRY}/${IMAGE}"
169 if [[ ! -z ${PUSH} ]]; then
170 docker tag ${IMAGE} ${PUSH}
175 echo "${IMAGE} pushed as ${PUSH} to Nexus"
179 # Verify all dependencies are available in PATH
181 for cmd in ${COMMANDS[*]}; do
182 command -v $cmd >/dev/null 2>&1 || FAILED_COMMANDS+=($cmd)
185 if [ ${#FAILED_COMMANDS[*]} -gt 0 ]; then
186 echo "Following commands where not found in PATH and are required:"
187 echo ${FAILED_COMMANDS[*]}
192 while [ "${1}" != "" ]; do
194 -d | --docker ) shift
195 NXS_DOCKER_IMG_LISTS+=("${1}")
197 -i | --input-directory ) shift
200 -ld | --load-docker-images ) DOCKER_LOAD="true"
203 NXS_NPM_LISTS+=("${1}")
205 -o | --output-directory ) shift
206 NEXUS_DATA_DIR="${1}"
209 NXS_PYPI_LISTS+=("${1}")
211 -rl | --resource-list-directory ) shift
221 # Setup directories with resources for docker, npm and pypi
222 NXS_SRC_DOCKER_IMG_DIR="${DATA_DIR}/offline_data/docker_images_for_nexus"
223 NXS_SRC_NPM_DIR="${DATA_DIR}/offline_data/npm_tar"
224 NXS_SRC_PYPI_DIR="${DATA_DIR}/offline_data/pypi"
226 # Setup specific resources lists
227 NXS_INFRA_LIST="${LISTS_DIR}/infra_docker_images.list"
228 NXS_DOCKER_IMG_LIST="${LISTS_DIR}/onap_docker_images.list"
229 NXS_RKE_DOCKER_IMG_LIST="${LISTS_DIR}/rke_docker_images.list"
230 NXS_NPM_LIST="${LISTS_DIR}/onap_npm.list"
231 NXS_PYPI_LIST="${LISTS_DIR}/onap_pip_packages.list"
233 # Setup Nexus image used for build and install infra
234 NEXUS_IMAGE="$(grep sonatype/nexus3 ${NXS_INFRA_LIST})"
235 NEXUS_IMAGE_TAR="${DATA_DIR}/offline_data/docker_images_infra/$(sed 's/\//\_/ ; s/$/\.tar/ ; s/\:/\_/' <<< ${NEXUS_IMAGE})"
237 # Set default lists if nothing specific defined by user
238 if [ $((${#NXS_DOCKER_IMG_LISTS[@]} + ${#NXS_NPM_LISTS[@]} + ${#NXS_PYPI_LISTS[@]})) -eq 0 ]; then
239 NXS_DOCKER_IMG_LISTS=("${NXS_DOCKER_IMG_LIST}" "${NXS_RKE_DOCKER_IMG_LIST}")
240 NXS_NPM_LISTS[0]="${NXS_NPM_LIST}"
241 NXS_PYPI_LISTS[0]="${NXS_PYPI_LIST}"
245 HOSTS_BACKUP="$(eval ${TIMESTAMP}_hosts.bk)"
246 cp /etc/hosts /etc/${HOSTS_BACKUP}
248 # Backup the current docker registry settings
249 if [ -f ~/.docker/config.json ]; then
250 DOCKER_CONF_BACKUP="$(eval ${TIMESTAMP}_config.json.bk)"
251 mv ~/.docker/config.json ~/.docker/${DOCKER_CONF_BACKUP}
254 # Setup default ports published to host as docker registry
255 PUBLISHED_PORTS="-p ${NEXUS_PORT}:${NEXUS_PORT} -p ${NEXUS_DOCKER_PORT}:${NEXUS_DOCKER_PORT}"
257 # Setup additional ports published to host based on simulated docker registries
258 # Setup simulated domain names to be able to push all to private Nexus repository
259 for DOCKER_IMG_LIST in "${NXS_DOCKER_IMG_LISTS[@]}"; do
260 publish_ports "${DOCKER_IMG_LIST}"
261 simulated_hosts "${DOCKER_IMG_LIST}"
264 # Nexus repository configuration setup
265 NEXUS_CONFIG_GROOVY='import org.sonatype.nexus.security.realm.RealmManager
266 import org.sonatype.nexus.repository.attributes.AttributesFacet
267 import org.sonatype.nexus.security.user.UserManager
268 import org.sonatype.nexus.repository.manager.RepositoryManager
269 import org.sonatype.nexus.security.user.UserNotFoundException
270 /* Use the container to look up some services. */
271 realmManager = container.lookup(RealmManager.class)
272 userManager = container.lookup(UserManager.class, "default") //default user manager
273 repositoryManager = container.lookup(RepositoryManager.class)
274 /* Managers are used when scripting api cannot. Note that scripting api can only create mostly, and that creation methods return objects of created entities. */
275 /* Perform cleanup by removing all repos and users. Realms do not need to be re-disabled, admin and anonymous user will not be removed. */
276 userManager.listUserIds().each({ id ->
277 if (id != "anonymous" && id != "admin")
278 userManager.deleteUser(id)
280 repositoryManager.browse().each {
281 repositoryManager.delete(it.getName())
283 /* Add bearer token realms at the end of realm lists... */
284 realmManager.enableRealm("NpmToken")
285 realmManager.enableRealm("DockerToken")
286 realmManager.enableRealm("PypiToken")
287 /* Create the docker user. */
288 security.addUser("docker", "docker", "docker", "docker@example.com", true, "docker", ["nx-anonymous"])
289 /* Create docker, npm and pypi repositories. Their default configuration should be compliant with our requirements, except the docker registry creation. */
290 repository.createNpmHosted("npm-private")
291 repository.createPyPiHosted("pypi-private")
292 def r = repository.createDockerHosted("onap", 8082, 0)
293 /* force basic authentication true by default, must set to false for docker repo. */
294 conf=r.getConfiguration()
295 conf.attributes("docker").set("forceBasicAuth", false)
296 repositoryManager.update(conf)'
298 # Prepare the Nexus configuration
299 NEXUS_CONFIG=$(echo "${NEXUS_CONFIG_GROOVY}" | jq -Rsc '{"name":"configure", "type":"groovy", "content":.}')
301 #################################
302 # Docker repository preparation #
303 #################################
305 if [ "${DOCKER_LOAD}" == "true" ]; then
306 # Load predefined Nexus image
307 docker load -i ${NEXUS_IMAGE_TAR}
308 # Load all necessary images
309 for DOCKER_IMG_LIST in "${NXS_DOCKER_IMG_LISTS[@]}"; do
310 load_docker_images "${DOCKER_IMG_LIST}"
314 ################################
315 # Nexus repository preparation #
316 ################################
318 # Prepare nexus-data directory
319 if [ -d ${NEXUS_DATA_DIR} ]; then
320 if [ "$(docker ps -q -f name="${NEXUS_DOMAIN}")" ]; then
321 echo "Removing container ${NEXUS_DOMAIN}"
322 docker rm -f $(docker ps -aq -f name="${NEXUS_DOMAIN}")
324 pushd ${NEXUS_DATA_DIR}/..
325 NXS_BACKUP="$(eval ${TIMESTAMP})_$(basename ${NEXUS_DATA_DIR})_bk"
326 mv ${NEXUS_DATA_DIR} "${NXS_BACKUP}"
327 echo "${NEXUS_DATA_DIR} already exists - backing up to ${NXS_BACKUP}"
331 mkdir -p ${NEXUS_DATA_DIR}
332 chown 200:200 ${NEXUS_DATA_DIR}
333 chmod 777 ${NEXUS_DATA_DIR}
335 # Save Nexus version to prevent/catch data incompatibility
336 docker images --no-trunc | grep sonatype/nexus3 | awk '{ print $1":"$2" "$3}' > ${NEXUS_DATA_DIR}/nexus.ver
339 NEXUS_CONT_ID=$(docker run -d --rm -v ${NEXUS_DATA_DIR}:/nexus-data:rw --name ${NEXUS_DOMAIN} ${PUBLISHED_PORTS} ${NEXUS_IMAGE})
340 echo "Waiting for Nexus to fully start"
341 until curl -su ${NEXUS_USERNAME}:${NEXUS_PASSWORD} http://${NEXUS_DOMAIN}:${NEXUS_PORT}/service/metrics/healthcheck | grep '"healthy":true' > /dev/null ; do
345 echo -e "\nNexus started"
347 # Configure the nexus repository
348 curl -sX POST --header 'Content-Type: application/json' --data-binary "${NEXUS_CONFIG}" http://${NEXUS_USERNAME}:${NEXUS_PASSWORD}@${NEXUS_DOMAIN}:${NEXUS_PORT}/service/rest/v1/script
349 curl -sX POST --header "Content-Type: text/plain" http://${NEXUS_USERNAME}:${NEXUS_PASSWORD}@${NEXUS_DOMAIN}:${NEXUS_PORT}/service/rest/v1/script/configure/run > /dev/null
351 ###########################
352 # Populate NPM repository #
353 ###########################
355 # Configure NPM registry to our Nexus repository
356 echo "Configure NPM registry to ${NPM_REGISTRY}"
357 npm config set registry "${NPM_REGISTRY}"
359 # Login to NPM registry
360 /usr/bin/expect <<EOF
363 send "${NEXUS_USERNAME}\n"
365 send "${NEXUS_PASSWORD}\n"
367 send "${NEXUS_EMAIL}\n"
371 # Patch problematic package
372 pushd ${NXS_SRC_NPM_DIR}
373 PATCHED_NPM="$(grep tsscmp ${NXS_NPM_LIST} | sed $'s/\r// ; s/\\@/\-/ ; s/$/\.tgz/')"
374 if [[ ! -z "${PATCHED_NPM}" ]] && ! zgrep -aq "${NPM_REGISTRY}" "${PATCHED_NPM}" 2>/dev/null; then
375 tar xzf "${PATCHED_NPM}"
376 rm -f "${PATCHED_NPM}"
377 sed -i 's|\"registry\":\ \".*\"|\"registry\":\ \"'"${NPM_REGISTRY}"'\"|g' package/package.json
378 tar -zcf "${PATCHED_NPM}" package
382 # Push NPM packages to Nexus repository
383 for NPM_LIST in "${NXS_NPM_LISTS[@]}"; do
384 push_npm "${NPM_LIST}"
388 ###############################
389 ## Populate PyPi repository #
390 ###############################
392 pushd ${NXS_SRC_PYPI_DIR}
393 for PYPI_LIST in "${NXS_PYPI_LISTS[@]}"; do
394 push_pip "${PYPI_LIST}"
398 ###############################
399 ## Populate Docker repository #
400 ###############################
402 # Login to simulated docker registries
403 # Push images to private nexus based on the list
404 # Images from default registry need to be tagged to private registry
405 # and those without defined repository in tag uses default repository 'library'
406 for DOCKER_IMG_LIST in "${NXS_DOCKER_IMG_LISTS[@]}"; do
407 docker_login "${DOCKER_IMG_LIST}"
408 push_docker "${DOCKER_IMG_LIST}"
411 ##############################
412 # Stop the Nexus and cleanup #
413 ##############################
415 echo "Stopping Nexus and returning backups"
418 docker stop ${NEXUS_CONT_ID} > /dev/null
420 # Return backed up configuration files
421 mv -f "/etc/${HOSTS_BACKUP}" /etc/hosts
423 if [ -f ~/.docker/${DOCKER_CONF_BACKUP} ]; then
424 mv -f ~/.docker/${DOCKER_CONF_BACKUP} ~/.docker/config.json
427 # Return default settings
428 npm config set registry "https://registry.npmjs.org"
430 echo "Nexus blob is built"