optimize size and time using "--no-cache-dir"

Using "--no-cache-dir" flag in pip install ,make sure dowloaded packages
by pip don't cached on system . This is a best practise which make sure
to fetch ftom repo instead of using local cached one . Further , in case
of Docker Containers , by restricing caching , we can reduce image size.
In term of stats , it depends upon the number of python packages
multiplied by their respective size . e.g for heavy packages with a lot
of dependencies it reduce a lot by don't caching pip packages.

Further , more detail information can be found at

https://medium.com/sciforce/strategies-of-docker-images-optimization-2ca9cc5719b6

Issue-ID: INT-1616

Signed-off-by: Pratik Raj <rajpratik71@gmail.com>
Change-Id: Id3e28faf35f36258362323b4a96bcf0f3f95726b
Signed-off-by: mrichomme <morgan.richomme@orange.com>

diff --git a/S3Ptools/locust_Grafana.sh b/S3Ptools/locust_Grafana.sh
index 5fa6fdc..5168f0e 100644 (file)
--- a/S3Ptools/locust_Grafana.sh
+++ b/S3Ptools/locust_Grafana.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 #grafana install for the use of locust
 # localgosh:80(grafana) & localhost:81
-pip install docker==3.1.4
+pip install --no-cache-dir docker==3.1.4
 git clone https://github.com/kamon-io/docker-grafana-graphite.git
 cd docker-grafana-graphite
 make up

diff --git a/deployment/aks/util/create_openstack_cli.sh b/deployment/aks/util/create_openstack_cli.sh
index 8f3f331..3e69ac4 100755 (executable)
--- a/deployment/aks/util/create_openstack_cli.sh
+++ b/deployment/aks/util/create_openstack_cli.sh
@@ -46,7 +46,7 @@ spec:
         apk add musl-dev && \
         apk add libffi-dev && \
         apk add openssl-dev && \
-        pip3 install python-openstackclient && \
+        pip3 install --no-cache-dir python-openstackclient && \
         sh -c 'echo ". /openstack/openstack_rc" >> /root/.profile; while true; do sleep 60; done;'
   restartPolicy: Never
   volumes:

diff --git a/test/hpa_automation/tosca/hpa_automation.py b/test/hpa_automation/tosca/hpa_automation.py
index 9302723..ac6a161 100755 (executable)
--- a/test/hpa_automation/tosca/hpa_automation.py
+++ b/test/hpa_automation/tosca/hpa_automation.py
@@ -3,7 +3,7 @@
 #Prerequisites for machine to run this
 #Put in required parameters in hpa_automation_config.json
 #Install python-pip (apt install python-pip)
-#Install python mysql.connector (pip install mysql-connector-python)
+#Install python mysql.connector (pip install --no-cache-dir mysql-connector-python)
 #Install ONAP CLI
 #Must have connectivity to the ONAP, a k8s vm already running is recommended
 #Create Preload File, the script will modify the parameters required from serivce model, service instance
@@ -223,16 +223,16 @@ def create_customer(parameters):
 def add_customer_subscription(parameters):
     subscription_check = 0
     for cloud_region, cloud_region_values in (parameters["cloud_region_data"]).iteritems():
-      if subscription_check == 0 :
-        subscription_string = "oclip subscription-create -x {} -c {} -z {} -e {} -y {} -r {} -m {} -u {} -p {}".format(\
-          parameters["customer_name"], cloud_region_values.get("tenant-id"), parameters["cloud-owner"], parameters["service_name"],\
-          cloud_region_values.get("default-tenant"), cloud_region, parameters["aai_url"], parameters["aai_username"], parameters["aai_password"] )
-      else:
-        subscription_string = "oclip subscription-cloud-add -x {} -c {} -z {} -e {} -y {} -r {} -m {} -u {} -p {}".format(\
-          parameters["customer_name"], cloud_region_values.get("tenant-id"), parameters["cloud-owner"], parameters["service_name"],\
-          cloud_region_values.get("default-tenant"), cloud_region, parameters["aai_url"], parameters["aai_username"], parameters["aai_password"] )
-      os.system(subscription_string)
-      subscription_check+=1
+        if subscription_check == 0 :
+            subscription_string = "oclip subscription-create -x {} -c {} -z {} -e {} -y {} -r {} -m {} -u {} -p {}".format(\
+            parameters["customer_name"], cloud_region_values.get("tenant-id"), parameters["cloud-owner"], parameters["service_name"],\
+            cloud_region_values.get("default-tenant"), cloud_region, parameters["aai_url"], parameters["aai_username"], parameters["aai_password"] )
+        else:
+            subscription_string = "oclip subscription-cloud-add -x {} -c {} -z {} -e {} -y {} -r {} -m {} -u {} -p {}".format(\
+            parameters["customer_name"], cloud_region_values.get("tenant-id"), parameters["cloud-owner"], parameters["service_name"],\
+            cloud_region_values.get("default-tenant"), cloud_region, parameters["aai_url"], parameters["aai_username"], parameters["aai_password"] )
+        os.system(subscription_string)
+        subscription_check+=1
 
 def register_vnfm_helper(vnfm_key, values, parameters):
     #Create vnfm
@@ -249,7 +249,7 @@ def register_vnfm(parameters):
     for vnfm_key, vnfm_values in vnfm_params.iteritems():
         register_vnfm_helper(vnfm_key, vnfm_values, parameters)
 
-def add_policy_models(parameters):
+def add_policy_models():
     mydb = mysql.connector.connect(
       host="policydb",
       user="policy_user",
@@ -361,15 +361,15 @@ def add_policy_models(parameters):
 def add_policies(parameters):
     #Loop through policy, put in resource_model_name and create policies
     for policy in os.listdir(parameters["policy_directory"]):
-      policy_name = "{}.{}".format(parameters["policy_scope"], os.path.splitext(policy)[0])
-      policy_file = (os.path.join(parameters["policy_directory"], policy))
-      #Create policy
-      os.system("oclip policy-create-outdated -m {} -u {} -p {} -x {} -S {} -T {} -o {} -b $(cat {})".format(parameters["policy_url"],\
-      parameters["policy_username"], parameters["policy_password"], policy_name, parameters["policy_scope"], \
-      parameters["policy_config_type"], parameters["policy_onapName"], policy_file))
-
-      #Push policy
-      os.system("oclip policy-push-outdated -m {} -u {} -p {} -x {} -b {} -c {}".format(parameters["policy_url"], \
+        policy_name = "{}.{}".format(parameters["policy_scope"], os.path.splitext(policy)[0])
+        policy_file = (os.path.join(parameters["policy_directory"], policy))
+        # Create policy
+        os.system("oclip policy-create-outdated -m {} -u {} -p {} -x {} -S {} -T {} -o {} -b $(cat {})".format(parameters["policy_url"],\
+        parameters["policy_username"], parameters["policy_password"], policy_name, parameters["policy_scope"], \
+        parameters["policy_config_type"], parameters["policy_onapName"], policy_file))
+
+        # Push policy
+        os.system("oclip policy-push-outdated -m {} -u {} -p {} -x {} -b {} -c {}".format(parameters["policy_url"], \
         parameters["policy_username"], parameters["policy_password"], policy_name, parameters["policy_config_type"],\
         parameters["policy_pdp_group"]))
 
@@ -521,7 +521,7 @@ else:
     print(ns_out)
 
 # 6.add_policies function not currently working, using curl commands
-add_policy_models(parameters)
+add_policy_models()
 add_policies(parameters)
 
 # 7. VFC part

diff --git a/test/mocks/datafilecollector-testharness/mr-sim/Dockerfile b/test/mocks/datafilecollector-testharness/mr-sim/Dockerfile
index c54713e..bb21a52 100755 (executable)
--- a/test/mocks/datafilecollector-testharness/mr-sim/Dockerfile
+++ b/test/mocks/datafilecollector-testharness/mr-sim/Dockerfile
@@ -4,6 +4,6 @@ COPY . /app
 
 WORKDIR /app
 
-RUN pip install -r requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
 
 EXPOSE 2222 2223

diff --git a/test/mocks/datafilecollector-testharness/mr-sim/setup.sh b/test/mocks/datafilecollector-testharness/mr-sim/setup.sh
index 2c94136..e6f50b2 100755 (executable)
--- a/test/mocks/datafilecollector-testharness/mr-sim/setup.sh
+++ b/test/mocks/datafilecollector-testharness/mr-sim/setup.sh
@@ -10,4 +10,4 @@ else
        virtualenv -p python3 .env
 fi
 
-source .env/bin/activate && pip3 install -r requirements.txt
+source .env/bin/activate && pip3 install --no-cache-dir -r requirements.txt

diff --git a/test/mocks/netconf-pnp-simulator/engine/configure-modules.sh b/test/mocks/netconf-pnp-simulator/engine/configure-modules.sh
index d40918f..4e1d17c 100755 (executable)
--- a/test/mocks/netconf-pnp-simulator/engine/configure-modules.sh
+++ b/test/mocks/netconf-pnp-simulator/engine/configure-modules.sh
@@ -81,7 +81,7 @@ create_python_venv()
     cd $env_dir
     # shellcheck disable=SC1091
     . ./bin/activate
-    pip install --requirement "$dir"/requirements.txt
+    pip install --no-cache-dir --requirement "$dir"/requirements.txt
   ) 1>&2
   echo $env_dir
 }

diff --git a/test/mocks/prov-mns-provider/Dockerfile b/test/mocks/prov-mns-provider/Dockerfile
index 748ad48..fef09b2 100644 (file)
--- a/test/mocks/prov-mns-provider/Dockerfile
+++ b/test/mocks/prov-mns-provider/Dockerfile
@@ -4,7 +4,7 @@ WORKDIR /app
 
 COPY src/requirements.txt ./
 
-RUN pip install -r requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
 
 COPY src /app
 

diff --git a/test/vcpe/bin/setup.sh b/test/vcpe/bin/setup.sh
index ad0e78c..30c25cb 100755 (executable)
--- a/test/vcpe/bin/setup.sh
+++ b/test/vcpe/bin/setup.sh
@@ -36,4 +36,4 @@ pip_setup_path="$(dirname $0)/../"
 
 curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
 python get-pip.py
-pip install -I ${pip_setup_path}
+pip install --no-cache-dir -I ${pip_setup_path}

diff --git a/test/vcpe/tox.ini b/test/vcpe/tox.ini
index 25785a6..8da9312 100644 (file)
--- a/test/vcpe/tox.ini
+++ b/test/vcpe/tox.ini
@@ -1,6 +1,6 @@
 # tox (https://tox.readthedocs.io/) is a tool for running tests
 # in multiple virtualenvs. This configuration file will run the
-# test suite on all supported python versions. To use it, "pip install tox"
+# test suite on all supported python versions. To use it, "pip install --no-cache-dir tox"
 # and then run "tox" from this directory.
 
 [tox]
@@ -10,6 +10,6 @@ envlist = pytest
 deps = pytest
 
 [testenv:pytest]
-basepython = python2
+basepython = python3
 commands =
     pytest

diff --git a/test/vcpe/vcpecommon.py b/test/vcpe/vcpecommon.py
index 971082c..0e02987 100755 (executable)
--- a/test/vcpe/vcpecommon.py
+++ b/test/vcpe/vcpecommon.py
@@ -10,7 +10,7 @@ import sys
 import ipaddress
 import mysql.connector
 import requests
-import commands
+import subprocess
 import time
 import yaml
 from novaclient import client as openstackclient
@@ -222,7 +222,7 @@ class VcpeCommon:
         self.logger.info('Adding vServer information to AAI for {0}'.format(openstack_stack_name))
         if not self.oom_mode:
             cmd = '/opt/demo.sh heatbridge {0} {1} vCPE'.format(openstack_stack_name, svc_instance_uuid)
-            ret = commands.getstatusoutput("ssh -i onap_dev root@{0} '{1}'".format(self.hosts['robot'], cmd))
+            ret = subprocess.getstatusoutput("ssh -i onap_dev root@{0} '{1}'".format(self.hosts['robot'], cmd))
             self.logger.debug('%s', ret)
         else:
             print('To add vGMUX vserver info to AAI, do the following:')