Improve python dependency management

- migrate from distutils to setuptools
- replace docker + local dependency files with requirements.txt
  that is used both locally and in docker
- remove separate `pip install`s from Dockerfiles and make sure that these deps are in the requirements file
  this eliminates the possibility for mismatches (between what is run in the tests and what is used in the build)

Issue-ID: CCSDK-4153
Change-Id: I9db055d05e8c66c9a83a71123492680259549542
Signed-off-by: Fiete Ostkamp <fiete.ostkamp@telekom.de>

diff --git a/ms/py-executor/requirements/test.txt b/ms/artifact-manager/requirements-test.txt
similarity index 73%
rename from ms/py-executor/requirements/test.txt
rename to ms/artifact-manager/requirements-test.txt
index 5304eac..0c1d9e2 100644 (file)
--- a/ms/py-executor/requirements/test.txt
+++ b/ms/artifact-manager/requirements-test.txt
@@ -1,3 +1,2 @@
 pytest==5.3.1
 pytest-grpc==0.7.0
--r local.txt
\ No newline at end of file

diff --git a/ms/artifact-manager/requirements/shared.txt b/ms/artifact-manager/requirements.txt
similarity index 100%
rename from ms/artifact-manager/requirements/shared.txt
rename to ms/artifact-manager/requirements.txt

diff --git a/ms/artifact-manager/requirements/docker.txt b/ms/artifact-manager/requirements/docker.txt
deleted file mode 100644 (file)
index 67f68dd..0000000
--- a/ms/artifact-manager/requirements/docker.txt
+++ /dev/null
@@ -1,2 +0,0 @@
--r shared.txt
-/opt/app/onap/dependencies/common
\ No newline at end of file

diff --git a/ms/artifact-manager/requirements/local.txt b/ms/artifact-manager/requirements/local.txt
deleted file mode 100644 (file)
index 6a06e16..0000000
--- a/ms/artifact-manager/requirements/local.txt
+++ /dev/null
@@ -1,2 +0,0 @@
--r shared.txt
-../../py-modules/common
\ No newline at end of file

diff --git a/ms/artifact-manager/setup.py b/ms/artifact-manager/setup.py
index b5b4487..6a96e27 100644 (file)
--- a/ms/artifact-manager/setup.py
+++ b/ms/artifact-manager/setup.py
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
-from distutils.core import setup
+from setuptools import setup
 
 
 setup(
@@ -22,4 +22,10 @@ setup(
     version="0.1",
     description="CDS Artifact Manager",
     packages=["manager"],
+    install_requires=[
+        "grpcio-tools>=1.25.0",
+        "protobuf>=3.20.1",
+        "onappylog>=1.0.9",
+        "click>=7.0",
+    ],
 )

diff --git a/ms/artifact-manager/tox.ini b/ms/artifact-manager/tox.ini
index a952678..fe7d744 100644 (file)
--- a/ms/artifact-manager/tox.ini
+++ b/ms/artifact-manager/tox.ini
@@ -6,7 +6,9 @@ setenv =
     PYTHONPATH = {toxinidir}
     CONFIGURATION = {toxinidir}/../configuration-local.ini
 deps =
-    -rrequirements/test.txt
+    -r{toxinidir}/requirements.txt
+    -r{toxinidir}/requirements-test.txt
+    {toxinidir}/../../py-modules/common
 commands = pytest
 [testenv:codelint]
 deps =
@@ -18,10 +20,12 @@ deps =
 commands = flake8 --doctest --docstring-convention google --max-line-length 120 --exclude .svn,CVS,.bzr,.hg,.git,__pycache__,.tox,.eggs,*.egg,*test.py --select=D {posargs:.}
 [testenv:coverage]
 basepython = python3.7
-setenv = 
+setenv =
     PYTHONPATH = {toxinidir}
     CONFIGURATION = {toxinidir}/../configuration-local.ini
 deps =
-    -rrequirements/test.txt
+    -r{toxinidir}/requirements.txt
+    -r{toxinidir}/requirements-test.txt
+    {toxinidir}/../../py-modules/common
     pytest-cov
 commands = pytest --cov=manager --cov-fail-under=60 --cov-config={toxinidir}/.coveragerc .

diff --git a/ms/command-executor/requirements.txt b/ms/command-executor/requirements.txt
new file mode 100644 (file)
index 0000000..b4331f4
--- /dev/null
+++ b/ms/command-executor/requirements.txt
@@ -0,0 +1,8 @@
+requests==2.26.0
+grpcio==1.20.0
+grpcio-tools==1.20.0
+virtualenv==16.7.9
+prometheus-client==0.11.0
+protobuf==3.20.1
+opentelemetry-distro==0.40b0
+opentelemetry-exporter-otlp-proto-http==1.19.0

diff --git a/ms/command-executor/src/main/docker/Dockerfile b/ms/command-executor/src/main/docker/Dockerfile
index 8405fe8..7c02cb4 100644 (file)
--- a/ms/command-executor/src/main/docker/Dockerfile
+++ b/ms/command-executor/src/main/docker/Dockerfile
@@ -18,8 +18,7 @@ RUN tar -xzf /source.tar.gz -C /tmp \
     && chmod -R 755 /opt \
     && apt-get update && apt-get install -y procps iputils-ping curl telnet && rm -rf /var/lib/apt/lists/* \
     && python -m pip install --no-cache-dir --upgrade pip setuptools \
-    && pip install --no-cache-dir requests==2.26.0 grpcio==1.20.0 grpcio-tools==1.20.0 virtualenv==16.7.9 prometheus-client==0.11.0 protobuf==3.20.1 \
-    && pip install --no-cache-dir opentelemetry-distro==0.40b0 opentelemetry-exporter-otlp-proto-http==1.19.0
+    && pip install --no-cache-dir -r /opt/app/onap/python/requirements.txt
 
 USER onap
 ENTRYPOINT /opt/app/onap/command-executor/start.sh

diff --git a/ms/command-executor/src/main/docker/distribution.xml b/ms/command-executor/src/main/docker/distribution.xml
index 7a16062..3384e52 100755 (executable)
--- a/ms/command-executor/src/main/docker/distribution.xml
+++ b/ms/command-executor/src/main/docker/distribution.xml
@@ -27,5 +27,12 @@
             <directory>${project.basedir}/src/main/python</directory>
             <outputDirectory>opt/app/onap/python</outputDirectory>
         </fileSet>
+        <fileSet>
+            <directory>${project.basedir}</directory>
+            <outputDirectory>opt/app/onap/python</outputDirectory>
+            <includes>
+                <include>requirements.txt</include>
+            </includes>
+        </fileSet>
     </fileSets>
-</assembly>
\ No newline at end of file
+</assembly>

diff --git a/ms/py-executor/docker/Dockerfile b/ms/py-executor/docker/Dockerfile
index a2c43b8..4271160 100644 (file)
--- a/ms/py-executor/docker/Dockerfile
+++ b/ms/py-executor/docker/Dockerfile
@@ -18,7 +18,10 @@ RUN tar -xzf /source.tar.gz -C /tmp \
     && chmod -R 755 /opt
 
 RUN python -m pip install --no-cache-dir --upgrade pip setuptools
-RUN pip install --no-cache-dir -r /opt/app/onap/python/requirements/docker.txt
+RUN pip install --no-cache-dir -r /opt/app/onap/python/requirements.txt
+RUN pip install --no-cache-dir /opt/app/onap/dependencies/common
+RUN pip install --no-cache-dir -r /opt/app/onap/dependencies/artifact-manager/requirements.txt
+RUN pip install --no-cache-dir /opt/app/onap/dependencies/artifact-manager
 
 RUN pip install --no-cache-dir opentelemetry-distro==0.40b0 opentelemetry-exporter-otlp-proto-http==1.19.0
 

diff --git a/ms/py-executor/docker/distribution.xml b/ms/py-executor/docker/distribution.xml
index a2739bd..af2bc2a 100755 (executable)
--- a/ms/py-executor/docker/distribution.xml
+++ b/ms/py-executor/docker/distribution.xml
@@ -62,7 +62,7 @@
             <directory>${project.basedir}</directory>
             <outputDirectory>opt/app/onap/python</outputDirectory>
             <includes>
-                <include>requirements/*.txt</include>
+                <include>requirements.txt</include>
             </includes>
             <useDefaultExcludes>true</useDefaultExcludes>
             <fileMode>0666</fileMode>

diff --git a/ms/artifact-manager/requirements/test.txt b/ms/py-executor/requirements-test.txt
similarity index 73%
rename from ms/artifact-manager/requirements/test.txt
rename to ms/py-executor/requirements-test.txt
index 5304eac..0c1d9e2 100644 (file)
--- a/ms/artifact-manager/requirements/test.txt
+++ b/ms/py-executor/requirements-test.txt
@@ -1,3 +1,2 @@
 pytest==5.3.1
 pytest-grpc==0.7.0
--r local.txt
\ No newline at end of file

diff --git a/ms/py-executor/requirements/shared.txt b/ms/py-executor/requirements.txt
similarity index 100%
rename from ms/py-executor/requirements/shared.txt
rename to ms/py-executor/requirements.txt

diff --git a/ms/py-executor/requirements/docker.txt b/ms/py-executor/requirements/docker.txt
deleted file mode 100644 (file)
index 58469c9..0000000
--- a/ms/py-executor/requirements/docker.txt
+++ /dev/null
@@ -1,4 +0,0 @@
--r shared.txt
-/opt/app/onap/dependencies/common
--r /opt/app/onap/dependencies/artifact-manager/requirements/docker.txt
-/opt/app/onap/dependencies/artifact-manager
\ No newline at end of file

diff --git a/ms/py-executor/requirements/local.txt b/ms/py-executor/requirements/local.txt
deleted file mode 100644 (file)
index 74cc76e..0000000
--- a/ms/py-executor/requirements/local.txt
+++ /dev/null
@@ -1,4 +0,0 @@
--r shared.txt
-../../py-modules/common
--r ../../artifact-manager/requirements/local.txt
-../artifact-manager
\ No newline at end of file

diff --git a/ms/py-executor/setup.py b/ms/py-executor/setup.py
new file mode 100644 (file)
index 0000000..7aba007
--- /dev/null
+++ b/ms/py-executor/setup.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+"""Copyright 2019 AT&T Intellectual Property.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from setuptools import setup, find_packages
+
+setup(
+    name="py-executor",
+    version="0.1",
+    description="CDS Python Executor",
+    packages=find_packages(exclude=["test", "test.*"]),
+    install_requires=[
+        "grpcio>=1.25.0",
+        "grpcio-tools>=1.25.0",
+        "protobuf>=3.20.1",
+        "configparser>=4.0.2",
+        "requests>=2.22.0",
+        "ncclient>=0.6.6",
+        "ansible>=2.8.5",
+    ],
+)

diff --git a/ms/py-executor/tox.ini b/ms/py-executor/tox.ini
index d722ed4..9af22c2 100644 (file)
--- a/ms/py-executor/tox.ini
+++ b/ms/py-executor/tox.ini
@@ -5,7 +5,11 @@ skipsdist=True
 setenv =
     CONFIGURATION = {toxinidir}/../configuration-local.ini
 deps =
-    -rrequirements/test.txt
+    -r{toxinidir}/requirements.txt
+    -r{toxinidir}/requirements-test.txt
+    {toxinidir}/../../py-modules/common
+    -r{toxinidir}/../artifact-manager/requirements.txt
+    {toxinidir}/../artifact-manager
 commands = pytest resource_resolution/
 [testenv:codelint]
 deps =
@@ -17,9 +21,13 @@ deps =
 commands = flake8 --doctest --docstring-convention google --max-line-length 120 --exclude .svn,CVS,.bzr,.hg,.git,__pycache__,.tox,.eggs,*.egg,*test.py --select=D {posargs:.}
 [testenv:coverage]
 basepython = python3.7
-setenv = 
+setenv =
     CONFIGURATION = {toxinidir}/../configuration-local.ini
 deps =
-    -rrequirements/test.txt
+    -r{toxinidir}/requirements.txt
+    -r{toxinidir}/requirements-test.txt
+    {toxinidir}/../../py-modules/common
+    -r{toxinidir}/../artifact-manager/requirements.txt
+    {toxinidir}/../artifact-manager
     pytest-cov
 commands = pytest --cov=manager --cov=resource_resolution --cov-fail-under=60 --cov-config={toxinidir}/.coveragerc resource_resolution/

diff --git a/py-modules/common/setup.py b/py-modules/common/setup.py
index 1ff59a6..f25d3e6 100644 (file)
--- a/py-modules/common/setup.py
+++ b/py-modules/common/setup.py
@@ -14,12 +14,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
-from distutils.core import setup
+from setuptools import setup
 
 setup(
     name="cdscommon",
     version="0.1",
     description="CDS Common Python Modules",
     packages=["proto"],
-    install_requires=["grpcio-tools"],
+    install_requires=["grpcio-tools>=1.25.0"],
 )