Enhance repo creation scripts idempotency by supporting caching

While 'yumdownloader' and 'apt-get download' commands do not
re-download packages that are already present in working dir,
scripts still need to perform package dependencies resolution
which is a time consuming activity hence a feature was added to
skip any further actions (download or deps resolution) on packages
that are already present locally.

For cold versus warm cache the gain in run time is:

       cold     warm
RPM     90s      10s
DEB     7m       90s

Change-Id: Ie502934cff6484844557832251e7ce0991789cfb
Issue-ID: OOM-2635
Signed-off-by: Bartek Grzybowski <b.grzybowski@partner.samsung.com>

diff --git a/build/create_repo.sh b/build/create_repo.sh
index e70d69b..f2bde32 100755 (executable)
--- a/build/create_repo.sh
+++ b/build/create_repo.sh
@@ -18,6 +18,9 @@ container_repo_volume="/mnt/repo/"
 # Additional packages lists files path within container
 container_list_volume="/mnt/additional-lists/"
 
+# Use cache by default
+drop_cache=false
+
 # Show script usage
 help () {
 cat <<EOF
@@ -34,6 +37,7 @@ usage: create_repo.sh [OPTION]...
   -t | --target-platform           target repository platform type (ubuntu/rhel/centos)
   -a | --additional-list           additional packages list; can be used multiple times for more additional lists
   -n | --container-name-suffix     add custom suffix to docker container name
+  -r | --drop-cache                remove cached packages (use package cache by default)
   -h | --help                      show this help
 
 If build folder from offline repository is not specified current one will be used by default.
@@ -82,24 +86,33 @@ do
             # Directory parameter
             # Set path to offline-installer build directory
             volume_offline_directory="$2"
+            shift
             ;;
         -d|--destination-repository)
             # Repository directory parameter
             # Set destination path for created repository
             volume_repo_directory="$2"
+            shift
             ;;
         -t|--target-platform)
             # Repository type (rpm/deb)
             # Set target platform for repository
             target_input="$2"
+            shift
             ;;
         -a|--additional-list)
             # Array of additional packages lists
             additional_lists+=("$2")
+            shift
             ;;
         -n|--container-name-suffix)
             # Set custom container name suffix
             container_name_suffix="_${2}"
+            shift
+            ;;
+        -r|--drop-cache)
+            # Set flag to clean cache
+            drop_cache=true
             ;;
         *)
             # unknown option
@@ -107,7 +120,7 @@ do
             exit 1
             ;;
     esac
-    shift;shift
+    shift
 done
 
 # Check if user specified repository type
@@ -148,6 +161,10 @@ then
     param_array+=(--directory ${container_repo_volume})
     param_array+=(--list ${container_offline_volume}data_lists/)
     param_array+=(--packages-lists-path ${container_list_volume})
+    if ${drop_cache};
+    then
+        param_array+=(--drop-cache)
+    fi
     [[ ! ${#additional_lists[@]} -eq 0 ]] && \
         for array_list in "${additional_lists[@]}";
         do
@@ -155,7 +172,7 @@ then
             mounted_lists+=(-v ${array_list}:${container_list_volume}${array_list##*/})
         done
 
-    docker run --name $container_name \
+        docker run --name $container_name \
                -v ${volume_offline_directory}:${container_offline_volume} \
                -v ${volume_repo_directory}:${container_repo_volume} \
                "${mounted_lists[@]}" \

diff --git a/build/docker-entrypoint.sh b/build/docker-entrypoint.sh
index 7d5f5b2..81dcd84 100755 (executable)
--- a/build/docker-entrypoint.sh
+++ b/build/docker-entrypoint.sh
@@ -26,6 +26,9 @@ PCKG_LIST_DIR=""
 # Path to additional packages lists
 ADD_LIST_DIR=""
 
+# Use cache by default
+drop_cache=false
+
 # Show help
 help () {
 cat <<EOF
@@ -37,6 +40,7 @@ usage: create-repo.sh [OPTION]...
   -l | --list                   input rpm/deb list directory
   -a | --additional-list        additional packages list; can be used multiple times
   -p | --packages-lists-path    other additional packages lists
+  -r | --drop-cache             remove cached packages (use package cache by default)
   -h | --help                   show this help
 
 Both paths have to be set with shared volume between
@@ -63,20 +67,28 @@ do
             # Directory parameter
             # Set target reposity path
             OFFLINE_REPO_DIR="$2"
+            shift
             ;;
         -l|--list)
             # List parameter
             # Set path containing onap_rpm.list or onap_deb.list file
             PCKG_LIST_DIR="$2"
+            shift
             ;;
         -p|--packages-lists-path)
             # Path parameter
             # Set path for additional packages lists
             ADD_LIST_DIR="$2"
+            shift
             ;;
         -a|--additional-list)
             # Array of additional packages lists
             ADDITIONAL_LISTS+=("$2")
+            shift
+            ;;
+        -r|--drop-cache)
+            # Set flag to clean cache
+            drop_cache=true
             ;;
         *)
             # unknown option
@@ -84,7 +96,7 @@ do
             exit
             ;;
     esac
-    shift;shift
+    shift
 done
 
 # Testing if directory parameter was used
@@ -98,18 +110,21 @@ fi
 # If not variable is set to default value /tmp/offline/data-list
 if test -z "$PCKG_LIST_DIR"
 then
-    PCKG_LIST_DIR="/tmp/offline/data_list/"
+    PCKG_LIST_DIR="/tmp/offline/data_list"
 fi
 
 # Testing if additional packages list parameter was used
 # If not variable is set to default value /tmp/additional-lists
 if test -z "$PCKG_LIST_DIR"
 then
-    PCKG_LIST_DIR="/tmp/additional-lists/"
+    PCKG_LIST_DIR="/tmp/additional-lists"
 fi
 
-# Clean target repo dir for idempotency
-rm -rf ${OFFLINE_REPO_DIR}/*
+# Clean target repo dir if --drop-cache set
+if ${drop_cache};
+then
+    rm -rf ${OFFLINE_REPO_DIR}/*
+fi
 
 case "$distro_type" in
     ubuntu)
@@ -133,22 +148,54 @@ case "$distro_type" in
         # https://bugs.launchpad.net/ubuntu/+source/aptitude/+bug/1543280
         chown _apt $OFFLINE_REPO_DIR
 
-        # Download all packages from onap_deb.list via apt-get to repository folder
-        for i in $(cat ${PCKG_LIST_DIR}onap_deb.list | awk '{print $1}');do apt-get download $i -y; done
-        for i in $(cat ${PCKG_LIST_DIR}onap_deb.list | awk '{print $1}');
-                    do
-                    for depends in $(apt-cache depends $i | grep -E 'Depends' | cut -d ':' -f 2,3 | sed -e s/'<'/''/ -e s/'>'/''/);
-                        do apt-get download $depends -y;
-                    done;
-                done
+        # Create tmp file for package list
+        list_file=$(mktemp)
+
+        # Enumerate packages that are already downloaded
+        for package in $(cat ${PCKG_LIST_DIR}/onap_deb.list);
+        do
+            # If package name contains explicit version info cut the version string off for further processing
+            p=$(echo $package |sed -r 's/=.*//')
+            # Add package to download list only if it's not already there
+            if [ $(ls ${p}_*.deb 2>/dev/null | wc -l) -eq 0 ];
+            then
+                echo ${package} >> ${list_file}
+            fi
+        done
+
+        # Download all packages via apt-get to repository folder
+        for i in $(cat ${list_file});do echo apt-get download $i -y; done
+        for i in $(cat ${list_file});
+            do
+                for depends in $(apt-cache depends $i | grep -E 'Depends' | grep -v 'Depends:.*>$' | cut -d ':' -f 2,3 | sed -e s/'<'/''/ -e s/'>'/''/);
+                do
+echo                    apt-get download $depends -y;
+                done;
+            done
 
         # Download all packages with dependencies from all additional packages lists via apt-get to repository folder
         if ! [ ${#ADDITIONAL_LISTS[@]} -eq 0 ]; then
             for list in ${ADDITIONAL_LISTS[@]}
             do
-                for i in $(cat ${ADD_LIST_DIR}$list | awk '{print $1}');do apt-get download $i -y; done
-                for i in $(cat ${ADD_LIST_DIR}$list | awk '{print $1}');
-                    do
+
+                # Create tmp file for package list
+                list_file=$(mktemp)
+
+                # Enumerate packages that are already downloaded
+                for package in $(cat ${ADD_LIST_DIR}/${list});
+                do
+                    # If package name contains explicit version info cut the version string off for further processing
+                    p=$(echo $package |sed -r 's/=.*//')
+                    # Add package to download list only if it's not already there
+                    if [ $(ls ${p}_*.deb 2>/dev/null | wc -l) -eq 0 ];
+                    then
+                        echo ${package} >> ${list_file}
+                    fi
+                done
+
+                for i in $(cat ${list_file});do apt-get download $i -y; done
+                for i in $(cat ${list_file});
+                do
                     for depends in $(apt-cache depends $i | grep -E 'Depends' | cut -d ':' -f 2,3 | sed -e s/'<'/''/ -e s/'>'/''/);
                         do apt-get download $depends -y;
                     done;
@@ -169,14 +216,42 @@ case "$distro_type" in
         # Add official docker repository
         yum-config-manager --add-repo=https://download.docker.com/linux/centos/7/x86_64/stable/
 
+        # Create tmp file for package list
+        list_file=$(mktemp)
+
+        # Enumerate packages that are already downloaded
+        for package in $(cat ${PCKG_LIST_DIR}/onap_rpm.list);
+        do
+            # Add package to download list only if it's not already there
+            if [ ! -f ${OFFLINE_REPO_DIR}/${package}.rpm ];
+            then
+                echo ${package} >> ${list_file}
+            fi
+        done
+
         # Download all packages from onap_rpm.list via yumdownloader to repository folder
-        for i in $(cat ${PCKG_LIST_DIR}onap_rpm.list | awk '{print $1}');do yumdownloader --resolve --downloadonly --destdir=${OFFLINE_REPO_DIR} $i -y; done
+        for i in $(cat ${list_file});do yumdownloader --resolve --downloadonly --destdir=${OFFLINE_REPO_DIR} $i -y; done
 
-        # Download all packages from all additional packages lists via apt-get to repository folder
+        # Download all packages from all additional packages lists via yumdownloader to repository folder
         if ! [ ${#ADDITIONAL_LISTS[@]} -eq 0 ]; then
             for list in ${ADDITIONAL_LISTS[@]}
             do
-                for i in $(cat ${ADD_LIST_DIR}$list | awk '{print $1}');do yumdownloader --resolve --downloadonly --destdir=${OFFLINE_REPO_DIR} $i -y; done
+                # Create tmp file for additional package list
+                list_file=$(mktemp)
+                # Enumerate packages that are already downloaded
+                for package in $(cat ${ADD_LIST_DIR}/${list});
+                do
+                    # Add package to download list only if it's not already there
+                    if [ ! -f ${OFFLINE_REPO_DIR}/${package}.rpm ];
+                    then
+                        echo ${package} >> ${list_file}
+                    fi
+                done
+
+                for i in $(cat ${list_file});
+                do
+                    yumdownloader --resolve --downloadonly --destdir=${OFFLINE_REPO_DIR} $i -y
+                done
             done
         fi