3 # COPYRIGHT NOTICE STARTS HERE
5 # Copyright 2018-2019 © Samsung Electronics Co., Ltd.
7 # Licensed under the Apache License, Version 2.0 (the "License");
8 # you may not use this file except in compliance with the License.
9 # You may obtain a copy of the License at
11 # http://www.apache.org/licenses/LICENSE-2.0
13 # Unless required by applicable law or agreed to in writing, software
14 # distributed under the License is distributed on an "AS IS" BASIS,
15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
19 # COPYRIGHT NOTICE ENDS HERE
21 ### This script prepares Nexus repositories data blobs for ONAP
23 ## The script requires following dependencies are installed: nodejs, jq, docker, twine, expect
24 ## All required resources are expected in the upper directory created during
25 ## download procedure as DATA_DIR or in the directory given as --input-directory
26 ## All lists used must be in project data_lists directory or in the directory given
27 ## as --resource-list-directory
32 TIMESTAMP="date +'%Y-%m-%d_%H-%M-%S'"
33 SCRIPT_LOG="/tmp/$(basename $0)_$(eval ${TIMESTAMP}).log"
36 exec &> >(tee -a "${SCRIPT_LOG}")
38 # Nexus repository location
41 NEXUS_DOCKER_PORT="8082"
42 NPM_REGISTRY="http://${NEXUS_DOMAIN}:${NEXUS_PORT}/repository/npm-private/"
43 PYPI_REGISTRY="http://${NEXUS_DOMAIN}:${NEXUS_PORT}/repository/pypi-private/"
44 DOCKER_REGISTRY="${NEXUS_DOMAIN}:${NEXUS_DOCKER_PORT}"
45 DEFAULT_REGISTRY="docker.io"
47 # Nexus repository credentials
49 NEXUS_PASSWORD=admin123
50 NEXUS_EMAIL=admin@example.org
53 LOCAL_PATH="$(readlink -f $(dirname ${0}))"
57 DATA_DIR="$(realpath ${LOCAL_PATH}/../../resources)"
58 NEXUS_DATA_DIR="${DATA_DIR}/nexus_data"
59 LISTS_DIR="${LOCAL_PATH}/data_lists"
61 # Required dependencies
62 COMMANDS=(jq docker expect npm twine)
65 echo " Example usage: build_nexus_blob.sh --input-directory </path/to/downloaded/files/dir> --output-directory
66 </path/to/output/dir> --resource-list-directory </path/to/dir/with/resource/list> [--load-docker-images]
68 -i | --input-directory directory containing file needed to create nexus blob. The structure of this directory must organized as described in build guide
69 -ld | --load-docker-images load docker images from stored files in the input directory
70 -o | --output-directory
71 -rl | --resource-list-directory directory with files containing docker, pypi and npm lists
76 load_docker_images () {
77 for ARCHIVE in $(sed $'s/\r// ; /^#/d ; s/\:/\_/g ; s/\//\_/g ; s/$/\.tar/g' ${1} | awk '{ print $1 }'); do
78 docker load -i ${NXS_SRC_DOCKER_IMG_DIR}/${ARCHIVE}
82 # Verify all dependencies are available in PATH
84 for cmd in ${COMMANDS[*]};
86 command -v $cmd >/dev/null 2>&1 || FAILED_COMMANDS+=($cmd)
88 if [ ${#FAILED_COMMANDS[*]} -gt 0 ];
90 echo "Following commands where not found in PATH and are required:"
91 echo ${FAILED_COMMANDS[*]}
96 while [ "${1}" != "" ]; do
98 -i | --input-directory ) shift
101 -ld | --load-docker-images ) DOCKER_LOAD="true"
103 -o | --output-directory ) shift
104 NEXUS_DATA_DIR="${1}"
106 -rl | --resource-list-directory ) shift
116 # Setup directories with resources for docker, npm and pypi
117 NXS_SRC_DOCKER_IMG_DIR="${DATA_DIR}/offline_data/docker_images_for_nexus"
118 NXS_SRC_NPM_DIR="${DATA_DIR}/offline_data/npm_tar"
119 NXS_SRC_PYPI_DIR="${DATA_DIR}/offline_data/pypi"
121 # Setup specific resources lists
122 NXS_DOCKER_IMG_LIST="${LISTS_DIR}/onap_docker_images.list"
123 NXS_NPM_LIST="${LISTS_DIR}/onap_npm.list"
124 NXS_PYPI_LIST="${LISTS_DIR}/onap_pip_packages.list"
126 # Setup Nexus image used for build and install infra
127 INFRA_LIST="${LISTS_DIR}/infra_docker_images.list"
128 NEXUS_IMAGE="$(grep sonatype/nexus3 ${INFRA_LIST})"
129 NEXUS_IMAGE_TAR="${DATA_DIR}/offline_data/docker_images_infra/$(sed 's/\//\_/ ; s/$/\.tar/ ; s/\:/\_/' <<< ${NEXUS_IMAGE})"
131 # Setup default ports published to host as docker registry
132 PUBLISHED_PORTS="-p ${NEXUS_PORT}:${NEXUS_PORT} -p ${NEXUS_DOCKER_PORT}:${NEXUS_DOCKER_PORT}"
134 # Setup additional ports published to host based on simulated docker registries
135 for REGISTRY in $(sed -n '/\.[^/].*\//p' ${NXS_DOCKER_IMG_LIST} | sed -e 's/\/.*$//' | sort -u | grep -v ${DEFAULT_REGISTRY} || true); do
136 if [[ ${REGISTRY} != *":"* ]]; then
137 if [[ ${PUBLISHED_PORTS} != *"80:${NEXUS_DOCKER_PORT}"* ]]; then
138 PUBLISHED_PORTS="${PUBLISHED_PORTS} -p 80:${NEXUS_DOCKER_PORT}"
141 REGISTRY_PORT="$(sed 's/^.*\:\([[:digit:]]*\)$/\1/' <<< ${REGISTRY})"
142 if [[ ${PUBLISHED_PORTS} != *"${REGISTRY_PORT}:${NEXUS_DOCKER_PORT}"* ]]; then
143 PUBLISHED_PORTS="${PUBLISHED_PORTS} -p ${REGISTRY_PORT}:${NEXUS_DOCKER_PORT}"
148 # Setup simulated domain names to be able to push all to private Nexus repository
149 SIMUL_HOSTS="$(sed -n '/\.[^/].*\//p' ${NXS_DOCKER_IMG_LIST} | sed -e 's/\/.*$// ; s/:.*$//' | sort -u | grep -v ${DEFAULT_REGISTRY} || true) ${NEXUS_DOMAIN}"
151 # Nexus repository configuration setup
152 NEXUS_CONFIG_GROOVY='import org.sonatype.nexus.security.realm.RealmManager
153 import org.sonatype.nexus.repository.attributes.AttributesFacet
154 import org.sonatype.nexus.security.user.UserManager
155 import org.sonatype.nexus.repository.manager.RepositoryManager
156 import org.sonatype.nexus.security.user.UserNotFoundException
157 /* Use the container to look up some services. */
158 realmManager = container.lookup(RealmManager.class)
159 userManager = container.lookup(UserManager.class, "default") //default user manager
160 repositoryManager = container.lookup(RepositoryManager.class)
161 /* Managers are used when scripting api cannot. Note that scripting api can only create mostly, and that creation methods return objects of created entities. */
162 /* Perform cleanup by removing all repos and users. Realms do not need to be re-disabled, admin and anonymous user will not be removed. */
163 userManager.listUserIds().each({ id ->
164 if (id != "anonymous" && id != "admin")
165 userManager.deleteUser(id)
167 repositoryManager.browse().each {
168 repositoryManager.delete(it.getName())
170 /* Add bearer token realms at the end of realm lists... */
171 realmManager.enableRealm("NpmToken")
172 realmManager.enableRealm("DockerToken")
173 realmManager.enableRealm("PypiToken")
174 /* Create the docker user. */
175 security.addUser("docker", "docker", "docker", "docker@example.com", true, "docker", ["nx-anonymous"])
176 /* Create docker, npm and pypi repositories. Their default configuration should be compliant with our requirements, except the docker registry creation. */
177 repository.createNpmHosted("npm-private")
178 repository.createPyPiHosted("pypi-private")
179 def r = repository.createDockerHosted("onap", 8082, 0)
180 /* force basic authentication true by default, must set to false for docker repo. */
181 conf=r.getConfiguration()
182 conf.attributes("docker").set("forceBasicAuth", false)
183 repositoryManager.update(conf)'
185 # Prepare the Nexus configuration
186 NEXUS_CONFIG=$(echo "${NEXUS_CONFIG_GROOVY}" | jq -Rsc '{"name":"configure", "type":"groovy", "content":.}')
188 #################################
189 # Prepare the local environment #
190 #################################
192 # Add simulated domain names to /etc/hosts
193 HOSTS_BACKUP="$(eval ${TIMESTAMP}_hosts.bk)"
194 cp /etc/hosts "/etc/${HOSTS_BACKUP}"
195 for DNS in ${SIMUL_HOSTS}; do
196 echo "127.0.0.1 ${DNS}" >> /etc/hosts
199 # Backup the current docker registry settings
200 if [ -f ~/.docker/config.json ]; then
201 DOCKER_CONF_BACKUP="$(eval ${TIMESTAMP}_config.json.bk)"
202 mv ~/.docker/config.json "~/.docker/${DOCKER_CONF_BACKUP}"
205 #################################
206 # Docker repository preparation #
207 #################################
209 if [ "${DOCKER_LOAD}" == "true" ]; then
210 # Load predefined Nexus image
211 docker load -i ${NEXUS_IMAGE_TAR}
212 # Load all necessary images
213 load_docker_images ${NXS_DOCKER_IMG_LIST}
216 ################################
217 # Nexus repository preparation #
218 ################################
220 # Prepare nexus-data directory
221 if [ -d ${NEXUS_DATA_DIR} ]; then
222 if [ "$(docker ps -q -f name="${NEXUS_DOMAIN}")" ]; then
223 echo "Removing container ${NEXUS_DOMAIN}"
224 docker rm -f $(docker ps -aq -f name="${NEXUS_DOMAIN}")
226 pushd ${NEXUS_DATA_DIR}/..
227 NXS_BACKUP="$(eval ${TIMESTAMP})_$(basename ${NEXUS_DATA_DIR})_bk"
228 mv ${NEXUS_DATA_DIR} "${NXS_BACKUP}"
229 echo "${NEXUS_DATA_DIR} already exists - backing up to ${NXS_BACKUP}"
233 mkdir -p ${NEXUS_DATA_DIR}
234 chown 200:200 ${NEXUS_DATA_DIR}
235 chmod 777 ${NEXUS_DATA_DIR}
237 # Save Nexus version to prevent/catch data incompatibility
238 docker images --no-trunc | grep sonatype/nexus3 | awk '{ print $1":"$2" "$3}' > ${NEXUS_DATA_DIR}/nexus.ver
241 NEXUS_CONT_ID=$(docker run -d --rm -v ${NEXUS_DATA_DIR}:/nexus-data:rw --name ${NEXUS_DOMAIN} ${PUBLISHED_PORTS} ${NEXUS_IMAGE})
242 echo "Waiting for Nexus to fully start"
243 until curl -su ${NEXUS_USERNAME}:${NEXUS_PASSWORD} http://${NEXUS_DOMAIN}:${NEXUS_PORT}/service/metrics/healthcheck | grep '"healthy":true' > /dev/null ; do
247 echo -e "\nNexus started"
249 # Configure the nexus repository
250 curl -sX POST --header 'Content-Type: application/json' --data-binary "${NEXUS_CONFIG}" http://${NEXUS_USERNAME}:${NEXUS_PASSWORD}@${NEXUS_DOMAIN}:${NEXUS_PORT}/service/rest/v1/script
251 curl -sX POST --header "Content-Type: text/plain" http://${NEXUS_USERNAME}:${NEXUS_PASSWORD}@${NEXUS_DOMAIN}:${NEXUS_PORT}/service/rest/v1/script/configure/run > /dev/null
253 ###########################
254 # Populate NPM repository #
255 ###########################
257 # Configure NPM registry to our Nexus repository
258 echo "Configure NPM registry to ${NPM_REGISTRY}"
259 npm config set registry "${NPM_REGISTRY}"
261 # Login to NPM registry
262 /usr/bin/expect <<EOF
265 send "${NEXUS_USERNAME}\n"
267 send "${NEXUS_PASSWORD}\n"
269 send "${NEXUS_EMAIL}\n"
273 # Patch problematic package
274 pushd ${NXS_SRC_NPM_DIR}
275 PATCHED_NPM="$(grep tsscmp ${NXS_NPM_LIST} | sed $'s/\r// ; s/\\@/\-/ ; s/$/\.tgz/')"
276 if [[ ! -z "${PATCHED_NPM}" ]] && ! zgrep -aq "${NPM_REGISTRY}" "${PATCHED_NPM}" 2>/dev/null; then
277 tar xzf "${PATCHED_NPM}"
278 rm -f "${PATCHED_NPM}"
279 sed -i 's|\"registry\":\ \".*\"|\"registry\":\ \"'"${NPM_REGISTRY}"'\"|g' package/package.json
280 tar -zcf "${PATCHED_NPM}" package
284 # Push NPM packages to Nexus repository
285 for ARCHIVE in $(sed $'s/\r// ; s/\\@/\-/g ; s/$/\.tgz/g' ${NXS_NPM_LIST});do
286 npm publish --access public ${ARCHIVE} > /dev/null
287 echo "NPM ${ARCHIVE} pushed to Nexus"
291 ###############################
292 ## Populate PyPi repository #
293 ###############################
295 pushd ${NXS_SRC_PYPI_DIR}
296 for PACKAGE in $(sed $'s/\r//; s/==/-/' ${NXS_PYPI_LIST}); do
297 twine upload -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" --repository-url ${PYPI_REGISTRY} ${PACKAGE}*
298 echo "PYPI ${PACKAGE} pushed to Nexus"
302 ###############################
303 ## Populate Docker repository #
304 ###############################
306 # Login to simulated docker registries
307 for REGISTRY in $(sed -n '/\.[^/].*\//p' ${NXS_DOCKER_IMG_LIST} | sed -e 's/\/.*$//' | sort -u | grep -v ${DEFAULT_REGISTRY}) ${DOCKER_REGISTRY}; do
308 echo "Docker login to ${REGISTRY}"
309 docker login -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" ${REGISTRY} > /dev/null
312 # Push images to private nexus based on the list
313 # Images from default registry need to be tagged to private registry
314 # and those without defined repository in tag uses default repository 'library'
315 for IMAGE in $(sed $'s/\r// ; /^#/d' ${NXS_DOCKER_IMG_LIST} | awk '{ print $1 }'); do
317 if [[ ${IMAGE} != *"/"* ]]; then
318 PUSH="${DOCKER_REGISTRY}/library/${IMAGE}"
319 elif [[ ${IMAGE} == *"${DEFAULT_REGISTRY}"* ]]; then
320 if [[ ${IMAGE} == *"/"*"/"* ]]; then
321 PUSH="$(sed 's/'"${DEFAULT_REGISTRY}"'/'"${DOCKER_REGISTRY}"'/' <<< ${IMAGE})"
323 PUSH="$(sed 's/'"${DEFAULT_REGISTRY}"'/'"${DOCKER_REGISTRY}"'\/library/' <<< ${IMAGE})"
325 elif [[ -z $(sed -n '/\.[^/].*\//p' <<< ${IMAGE}) ]]; then
326 PUSH="${DOCKER_REGISTRY}/${IMAGE}"
328 if [[ ! -z ${PUSH} ]]; then
329 docker tag ${IMAGE} ${PUSH}
334 echo "${IMAGE} pushed as ${PUSH} to Nexus"
337 ##############################
338 # Stop the Nexus and cleanup #
339 ##############################
341 echo "Stopping Nexus and returning backups"
344 docker stop ${NEXUS_CONT_ID} > /dev/null
346 # Return backed up configuration files
347 mv -f "/etc/${HOSTS_BACKUP}" /etc/hosts
349 if [ -f "~/.docker/${DOCKER_CONF_BACKUP}" ]; then
350 mv -f "~/.docker/${DOCKER_CONF_BACKUP}" ~/.docker/config.json
353 # Return default settings
354 npm config set registry "https://registry.npmjs.org"
356 echo "Nexus blob is built"